push file for api fetch

background.js

@@ -32,24 +32,27 @@ chrome.extension.onMessage.addListener(function(request,sender,sendResponse){
   }
   console.log("Current index : ", index);
   console.log("Current noImages : " , noImages);
-  if(index >= skus.length-1) {
+  if(index >= skus.length) {
     console.log("Last file was downloaded");
     alert(noImages);
+
+    index = 0;
+    noImages = "";
   }
 });
 
 chrome.downloads.onDeterminingFilename.addListener(function(item, suggest) {
 
-  if(!skus.length){
-    chrome.storage.local.get('idArray', function (items) {
-      skus = items.idArray.split(',');
-    });
-  }
+  // if(!skus.length){
+  //   chrome.storage.local.get('idArray', function (items) {
+  //     skus = items.idArray.split(',');
+  //   });
+  // }
   
   var new_filename = skus[index] + ".zip"; 
   suggest({filename: new_filename});
   index++;
-  if(index >= skus.length-1) {
+  if(index >= skus.length) {
     console.log("Last file was downloaded");
     alert(noImages);
   }

contentScript.js

@@ -3,18 +3,18 @@ var logined = false;
 
 //If at login site, fill in login infos
 if (link.match(/login/g)){
-    window.onload = function() {
-        document.querySelector("#login_input_user_name").value="myra@documentonready.com";
-        document.querySelector("#login_input_password").value="GUMyra12345";
-        document.querySelector(".jss70").click();
-    }
+    // window.onload = function() {
+    //     document.querySelector("#login_input_user_name").value="myra@documentonready.com";
+    //     document.querySelector("#login_input_password").value="GUMyra12345";
+    //     document.querySelector(".jss70").click();
+    // }
 //If logined, get the idArray and go to the first asset page
 } else if (link == "https://digitallibrary-external.fastretailing.com/" && !logined) {
-    logined = true;
-    chrome.storage.local.get('idArray', function (items) {
-        var a = items.idArray.split(',');
-        document.location.href = ("https://digitallibrary-external.fastretailing.com/asset?t=datacategory%2Fasset&w="+ a[0] + "&r=north");
-    });
+    // logined = true;
+    // chrome.storage.local.get('idArray', function (items) {
+    //     var a = items.idArray.split(',');
+    //     document.location.href = ("https://digitallibrary-external.fastretailing.com/asset?t=datacategory%2Fasset&w="+ a[0] + "&r=north");
+    // });
 //If at asset page, automate scrollDown and fliter process
 } else if (link.match(/asset\?/g)) {
     var a = [];

fetch/fetch.js

+(function(){
+
+    const baseurl = "https://digitallibrary-external.fastretailing.com/";
+    const fetchNumber = 30;
+    const sleepTime = 3000;
+    let allowDebugMessage = true;
+
+    function addLightBox(){
+        if(document.URL === baseurl){
+            fetch(chrome.extension.getURL("fetch/lightbox.html"))
+            .then(response => response.text())
+            .then(result => document.body.insertAdjacentHTML('beforeend', result))
+            .then(()=>{
+
+                // turn on/ off lightbox
+                document.querySelector("#lightbox-switch").addEventListener("click",function(){
+                    document.getElementById("fetch-lightbox").classList.add("on");
+                });
+                document.querySelector("#fetch-lightbox .close-btn").addEventListener("click",function(){
+                    document.getElementById("fetch-lightbox").classList.remove("on");
+                });
+
+                // operation
+                document.querySelector("#searchSKUs").addEventListener("click", startFetching);
+                document.querySelector("#downloadSelectedSKUs").addEventListener("click",downloadFetching);
+                document.querySelector("#downloadAllSKUs").addEventListener("click",extensionDownloading);
+                document.querySelector("#filter-date").addEventListener("change",filterByDate);
+ 
+                document.querySelector("#fetch-lightbox .debug-box").addEventListener("click",function(event){
+                    if(!event.target.classList.contains("debug-box")){
+                        event.target.closest('.asset-item').classList.toggle("on");
+                    }
+                });
+                
+            })
+            .catch(error => console.log('error', error));
+        }
+    }
+
+    /* Utility function */
+
+    function sleep(ms) {
+        return new Promise(resolve => setTimeout(resolve, ms));
+    }
+
+    function addDebugMessage(str){
+        if(allowDebugMessage){
+            let div = document.createElement("div");
+            let content = document.createTextNode(str);
+    
+            div.appendChild(content);
+            document.querySelector("#fetch-lightbox .debug-box").appendChild(div);
+        }else{
+            console.log(str);
+        }
+    }
+
+    function generateCookie(){
+        let cookie = document.cookie;
+        cookie += "; digitallibrary-session=" + document.getElementById("session-string").value;
+        cookie += "; digitallibrary-session.sig=" + document.getElementById("session-sig-string").value;
+        return cookie;
+    }
+
+    function getHeader(){
+        var header = new Headers();
+        header.append("authority", "digitallibrary-external.fastretailing.com");
+        header.append("pragma", "no-cache");
+        header.append("cache-control", "no-cache");
+        header.append("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36");
+        header.append("accept", "*/*");
+        header.append("accept-language", "en-US,en;q=0.9,zh-TW;q=0.8,zh;q=0.7,zh-CN;q=0.6,ja;q=0.5");
+        header.append("cookie", generateCookie());
+        return header;
+    }
+
+    function getConfig(header){
+        return {
+            "method": "GET",
+            "headers": header,
+            "mode": "cors",
+            "redirect": "follow"
+        };
+    }
+
+    /* End of utility function */
+
+    /* Start of main function */
+
+    async function startFetching(){
+        let SKUString = document.getElementById("sku-list").value;
+        if(SKUString.length){
+            let SKUlist = SKUString.split("\n");
+            let container = document.querySelector("#fetch-lightbox .debug-box");
+            allowDebugMessage = false;
+
+            while(container.hasChildNodes()){
+                container.removeChild(container.firstChild);
+            }
+
+            for(let SKU of SKUlist){
+                let assetlist = await fetchSKU(SKU);
+                assetlist = assetlist.filter(asset => asset.file_extension === "jpg" || asset.file_extension === "gif");
+                assetlist = assetlist.filter(asset => !asset.filename.includes("_I0"));
+
+                let DOMString = assetlist.map(asset => {
+                    return `
+                        <div class="asset-item">
+                            <div><img src="/api/asset/${asset.id}/thumbnail/medium?r=1" loading="lazy"/></div>
+                            <p>
+                                ${SKU}<br />
+                                ${asset.filename}.${asset.file_extension}
+                            </p>
+                            <span class="asset-id" hidden>${asset.id}</span>
+                            <span class="asset-version" hidden>${asset.fileVersion}</span>
+                            <span class="asset-date" hidden>${asset.updatedDate}</span>
+                        </div>
+                    `;
+                }).join('');
+                container.insertAdjacentHTML('beforeend', DOMString);
+                filterByDate();
+            }
+        }
+    }
+
+    async function downloadFetching(){
+        let DOMArray = [...document.querySelectorAll("#fetch-lightbox .debug-box .asset-item.on")];
+        let downloadlist = DOMArray.map(x => {
+            return{
+                "id" : x.querySelector(".asset-id").innerHTML,
+                "fileVersion" :  x.querySelector(".asset-version").innerHTML
+            };
+        });
+        await downloadList(downloadlist, `asset_file`);
+
+        // remove the clicked state on asset item
+        document.querySelectorAll("#fetch-lightbox .debug-box .asset-item.on").forEach(x => x.classList.remove("on"));
+    }
+
+    async function extensionDownloading(){
+        let SKUString = document.getElementById("sku-list").value;
+        if(SKUString.length){
+            let SKUlist = SKUString.split("\n");
+            let container = document.querySelector("#fetch-lightbox .debug-box");
+            allowDebugMessage = true;
+            while(container.hasChildNodes()){
+                container.removeChild(container.firstChild);
+            }
+
+            for(let SKU of SKUlist){
+                let assetlist = await fetchSKU(SKU);
+                let downloadlist = filtering(assetlist, SKU);
+                await downloadList(downloadlist["product"], `${SKU}_product`);
+                await sleep(sleepTime);
+                await downloadList(downloadlist["model"], `${SKU}_model`);
+                await sleep(sleepTime);
+            }
+        }
+    }
+
+    async function fetchSKU(SKU = ""){
+        if(SKU.length == 0){
+            return;
+        }
+
+        let currentms = new Date().getTime();
+        let url = `${baseurl}api/asset?time=${currentms}&t=datacategory%2Fasset&w=${SKU}&r=north&z=${fetchNumber}&c=DEFAULT`;
+        let header = getHeader();
+        let config = getConfig(header);
+
+        addDebugMessage(`Start fetching SKU=${SKU}`);
+
+        return await fetch(url,config)
+        .then(response => response.json())
+        .then(async function(result){
+
+            await sleep(sleepTime);
+
+            //get json data
+            let assetlist = result.assets;
+            let scrollId = result.scrollId;
+            let total = result.total;
+
+            // fetch the remaining item by simluating scroll operation
+            for(let i = total - fetchNumber ; i > 0 ; i -= fetchNumber){
+                addDebugMessage(`fetching SKU=${SKU} Process: ${i} remains`);
+
+                let response = await fetch(`${baseurl}api/asset/more?scrollId=${scrollId}`, config)
+                .then(response => response.json())
+                .catch(error => addDebugMessage(error));
+
+                assetlist = assetlist.concat(response.assets);
+
+                await sleep(sleepTime);
+            }
+            addDebugMessage(`fetching SKU=${SKU} completed, total item: ${total}`);
+
+            return assetlist;
+        })
+        .catch(error => console.error(error));
+    }
+
+    // filter asset by Date
+    // only used on Search and Download
+    function filterByDate(){
+        let datefilter = 0;
+        if(document.getElementById("filter-date").value){
+            datefilter = new Date().setHours(0,0,0,0) / 1000 - document.getElementById("filter-date").value * 86400;
+        }
+        document.querySelectorAll("#fetch-lightbox .debug-box .asset-item").forEach(asset => {
+            asset.classList.remove("on");
+            if(asset.querySelector(".asset-date").innerHTML > datefilter){
+                asset.style.display = "";
+            }else{
+                asset.style.display = "none";
+            }
+        });
+    }
+
+    // filter asset to different array
+    // only used on Download All SKUs
+    // usage:
+    //     let downloadlist = filtering(assetlist, SKU);
+
+    function filtering(assetlist, SKU){
+        assetlist = assetlist.filter(asset => asset.file_extension === "jpg" || asset.file_extension === "gif");
+
+        let multilist = assetlist.reduce(function(arrlist, item){
+
+            if( item.filename.includes(`goods_${SKU}_sub`) || 
+                item.filename.includes(`_${SKU}_chip`) || 
+                (item.filename.includes(SKU) && item.models.length === 0 && item.file.filesize <= 1048576) ){  // file name include SKU, no model and size <= 1MB
+                arrlist["product"].push(item);
+
+            }else if( (item.filename.includes(SKU) && item.models.length > 0) || (item.filename.includes("_M0"))){
+                arrlist["model"].push(item);
+            }
+            return arrlist;
+
+        }, {"product" : [], "model" : []}); // initial value
+
+        return multilist;
+    }
+
+    // given the asset id and file version to download the asset
+    // downloadList usage:
+    //    await downloadList(downloadlist["product"], "123456_product");
+    //    await downloadList(downloadlist["model"], "123456_model");
+    
+    async function downloadList(assetlist, name){
+        if(assetlist.length >= 2){
+
+            let id = assetlist.map(asset => asset.id);
+            let revision = assetlist.map(asset => asset.fileVersion);
+
+            let json = JSON.stringify({id : id, revision: revision});
+            console.log(json);
+
+            // prepare header for POST request
+            let postheader = getHeader();
+            postheader.append("Content-Type", "application/json");
+            postheader.append("csrf-token", document.querySelector('meta[name="csrf-token"]').content);
+            postheader.append("origin", "https://digitallibrary-external.fastretailing.com");
+
+            // send POST request for getting download token
+            let response = await fetch(`${baseurl}api/asset/downloadtoken`,{
+                "method": "POST",
+                "headers": postheader,
+                "mode": "cors",
+                "redirect": "follow",
+                "body": json
+            })
+            .then(response => response.json())
+            .catch(error => console.error(error));
+
+            // send GET request for downloading the zip file
+            fetch(`/api/asset/download/${response.token}`, getConfig(getHeader()))
+            .then(response => response.blob())
+            .then(blob => {
+
+                // https://stackoverflow.com/questions/32545632/how-can-i-download-a-file-using-window-fetch
+                let url = window.URL.createObjectURL(blob);
+                let a = document.createElement('a');
+                a.href = url;
+                a.download = `${name}.zip`;
+                document.body.appendChild(a); // we need to append the element to the dom -> otherwise it will not work in firefox
+                a.click();    
+                a.remove();  //afterwards we remove the element again     
+                
+                addDebugMessage(`download ${name}.zip completed`);
+            })
+            .catch(error => console.error(error));
+        
+        }else if(assetlist.length === 1){
+            let id = assetlist[0].id;
+            let filename = `${assetlist[0].filename}.${assetlist[0].file_extension}`
+            let downloadDOM = document.getElementById("download-trigger");
+            downloadDOM.setAttribute('href', `/api/asset/${id}/download/`);
+            downloadDOM.setAttribute('download', filename);
+            downloadDOM.click();
+
+            addDebugMessage(`download file ${filename} completed`);
+        }else{
+            addDebugMessage(`asset list is empty.`);
+        }
+    }
+
+    addLightBox();
+
+})();
\ No newline at end of file

fetch/lightbox.css

+#fetch-lightbox, #fetch-lightbox *{
+    box-sizing: border-box;
+}
+
+#fetch-lightbox{
+    width: 100%;
+    height: 100%;
+    position: fixed;
+    top: 0;
+    left: 0;
+
+    background-color: rgba(0,0,0,0.5);
+    padding: 50px;
+    
+    display: none;
+    z-index: 2500;
+}
+
+#fetch-lightbox.on{
+    display: block;
+}
+
+#fetch-lightbox .close-btn{
+    width: 30px;
+    height: 30px;
+    
+    position: fixed;
+    right: 15px;
+    top: 15px;
+}
+
+#fetch-lightbox .lightbox-content{
+    width: 100%;
+    height: 100%;
+    padding: 40px;
+    border-radius: 5px;
+    background-color: white;
+
+    display: grid;
+    grid-template-columns: 150px 1fr 250px;
+    grid-template-rows: 1fr 2fr;
+}
+
+#fetch-lightbox .lightbox-content > div{
+    padding: 15px;
+}
+
+#fetch-lightbox h6{
+    margin: 0 0 10px;
+    font-size: 1.2em;
+}
+
+#fetch-lightbox textarea{
+    width: 100px;
+    height: 80%;
+}
+
+#fetch-lightbox .input-box{
+    display: flex;
+    align-items: center;
+}
+
+#fetch-lightbox input{
+    flex-grow: 1;
+    height: 30px;
+    margin-bottom: 15px;
+}
+
+#fetch-lightbox label{
+    margin-bottom: 15px;
+    margin-right: 5px;
+}
+
+#fetch-lightbox .cookie-remark{
+    font-size: 0.8em;
+    margin-top: 0;
+    margin-bottom: 15px;
+}
+
+#fetch-lightbox .debug-box{
+    grid-column-start: 1;
+    grid-column-end: 4;
+    grid-row-start: 2;
+
+    border: 2px solid #cccccc;
+    border-radius: 5px;
+    overflow: scroll;
+}
+
+#fetch-lightbox .asset-item{
+    width: 200px;
+    margin: 3px;
+    padding: 10px;
+    
+    display: inline-block;
+    text-align: center;
+}
+
+#fetch-lightbox .asset-item.on{
+    background-color: #cccccc;
+}
+
+#fetch-lightbox .asset-item p{
+    margin: 5px 0;
+    font-size: 0.9em;
+}
+
+#fetch-lightbox .asset-item > div{
+    width: 100%;
+    height: 250px;
+    display: flex;
+    align-items: center;
+}
+
+#fetch-lightbox .asset-item img{
+    width: 100%;
+    max-height: 100%;
+    object-fit: contain;
+}
+
+
+#searchSKUs, #downloadSelectedSKUs, #downloadAllSKUs{
+    display: inline-block;
+    padding: 12px;
+    margin-bottom: 5px;
+    border: 2px solid transparent;
+    border-radius: 5px;
+    cursor: pointer;
+}
+
+#searchSKUs:hover, #downloadSelectedSKUs:hover, #downloadAllSKUs:hover{
+    border-color: black;
+}
+
+#searchSKUs{
+    background-color: hsl(40, 77%, 83%);
+}
+
+#downloadSelectedSKUs{
+    background-color: hsl(60, 77%, 83%);
+}
+
+#downloadAllSKUs{
+    background-color: hsl(80, 77%, 83%);
+}
+
+#lightbox-switch{
+    position: fixed;
+    right: 0px;
+    bottom: 0px;
+
+    background-color: white;
+    padding: 12px;
+    z-index: 1000;
+}
\ No newline at end of file

fetch/lightbox.html

+<div id="fetch-lightbox">
+    <div class="close-btn">
+       <svg version="1.1" viewBox="0 0 512 512" xml:space="preserve" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+           <path fill="white" d="M443.6,387.1L312.4,255.4l131.5-130c5.4-5.4,5.4-14.2,0-19.6l-37.4-37.6c-2.6-2.6-6.1-4-9.8-4c-3.7,0-7.2,1.5-9.8,4  L256,197.8L124.9,68.3c-2.6-2.6-6.1-4-9.8-4c-3.7,0-7.2,1.5-9.8,4L68,105.9c-5.4,5.4-5.4,14.2,0,19.6l131.5,130L68.4,387.1  c-2.6,2.6-4.1,6.1-4.1,9.8c0,3.7,1.4,7.2,4.1,9.8l37.4,37.6c2.7,2.7,6.2,4.1,9.8,4.1c3.5,0,7.1-1.3,9.8-4.1L256,313.1l130.7,131.1  c2.7,2.7,6.2,4.1,9.8,4.1c3.5,0,7.1-1.3,9.8-4.1l37.4-37.6c2.6-2.6,4.1-6.1,4.1-9.8C447.7,393.2,446.2,389.7,443.6,387.1z"/>
+        </svg>
+    </div>
+    <div class="lightbox-content">
+
+        <div class="sku-box">
+            <h6>SKU List:</h6>
+            <textarea id="sku-list" cols="20"></textarea>
+        </div>
+
+        <div class="cookie-box">
+            <h6>Cookie</h6>
+            <div class="input-box">
+                <label>digitallibrary-session: </label>
+                <input type="text" id="session-string" name="session" autocomplete="off"/>
+            </div>
+            <div class="input-box">
+                <label>digitallibrary-session.sig: </label>
+                <input type="text" id="session-sig-string" name="session-sig" autocomplete="off"/>
+            </div>
+
+            <p class="cookie-remark">
+                Manuel input required because session cannot obtain from javascript <br />
+                To copy the cookie string, go to:
+                Web Inspector > Application > Storage > Cookie
+            </p>
+        </div>
+
+        <div class="operation-box">
+            <h6>Search and Download</h6>
+            <div class="input-box">
+                <label>Date: </label>
+                <input type="text" id="filter-date" name="filter-date" value="7" >
+            </div>
+            <button id="searchSKUs">Search</button>
+            <button id="downloadSelectedSKUs">Download</button>
+
+            <h6>Download All SKUs</h6>
+            <button id="downloadAllSKUs">Download</button>
+
+
+            <a id="download-trigger" href="#" download hidden >Download</a>
+
+        </div>
+
+        <div class="debug-box">
+
+
+        </div>
+
+    </div>
+</div>
+<div id="lightbox-switch">Open Fetch Menu</div>
\ No newline at end of file

manifest.json

 {
     "name" : "Image Downloader",
 
-    "version" : "1.0",
+    "version" : "2.0",
 
-    "description" : "The extension for downloading images",
+    "description" : "The extension for downloading images - NEW",
 
     "background" : {
         "scripts" : ["background.js"],
@@ -29,6 +29,13 @@
         {
             "matches": ["<all_urls>"],
             "js": ["jquery.min.js","contentScript.js"]
+        },
+        {
+            "matches": ["https://digitallibrary-external.fastretailing.com/"],
+            "js": ["fetch/fetch.js"],
+            "css": ["fetch/lightbox.css"]
         }
-    ]
+    ],
+
+    "web_accessible_resources": ["fetch/lightbox.html"]
 }
\ No newline at end of file