support: artstation #91;

MapoMagpie · Sep 9, 2024 · bc6e4ba · bc6e4ba
1 parent 6b472e9
commit bc6e4ba
Show file tree

Hide file tree

Showing 8 changed files with 297 additions and 54 deletions.
diff --git a/eh-view-enhance.meta.js b/eh-view-enhance.meta.js
@@ -48,6 +48,7 @@
 // @match              https://*.copymanga.tv/*
 // @match              https://e621.net/*
 // @match              https://arca.live/*
+// @match              https://*.artstation.com/*
 // @require            https://cdn.jsdelivr.net/npm/@zip.js/[email protected]/dist/zip-full.min.js
 // @require            https://cdn.jsdelivr.net/npm/[email protected]/dist/FileSaver.min.js
 // @require            https://cdn.jsdelivr.net/npm/[email protected]/dist/pica.min.js
@@ -82,6 +83,7 @@
 // @connect            mangafuna.xyz
 // @connect            e621.net
 // @connect            namu.la
+// @connect            artstation.com
 // @connect            *
 // @grant              GM_getValue
 // @grant              GM_setValue

diff --git a/eh-view-enhance.user.js b/eh-view-enhance.user.js
@@ -48,6 +48,7 @@
 // @match              https://*.copymanga.tv/*
 // @match              https://e621.net/*
 // @match              https://arca.live/*
+// @match              https://*.artstation.com/*
 // @require            https://cdn.jsdelivr.net/npm/@zip.js/[email protected]/dist/zip-full.min.js
 // @require            https://cdn.jsdelivr.net/npm/[email protected]/dist/FileSaver.min.js
 // @require            https://cdn.jsdelivr.net/npm/[email protected]/dist/pica.min.js
@@ -82,6 +83,7 @@
 // @connect            mangafuna.xyz
 // @connect            e621.net
 // @connect            namu.la
+// @connect            artstation.com
 // @connect            *
 // @grant              GM_getValue
 // @grant              GM_setValue
@@ -1257,6 +1259,31 @@ Report issues here: <a target="_blank" href="https://github.com/MapoMagpie/eh-vi
       }, {}, 10 * 1e3);
     });
   }
+  async function batchFetch(urls, concurrency, respType = "text") {
+    const results = new Array(urls.length);
+    let i = 0;
+    while (i < urls.length) {
+      const batch = urls.slice(i, i + concurrency);
+      const batchPromises = batch.map(
+        (url, index) => window.fetch(url).then((resp) => {
+          if (resp.ok) {
+            switch (respType) {
+              case "text":
+                return resp.text();
+              case "json":
+                return resp.json();
+              case "arraybuffer":
+                return resp.arrayBuffer();
+            }
+          }
+          throw new Error(`failed to fetch ${url}: ${resp.status} ${resp.statusText}`);
+        }).then((raw) => results[index + i] = raw)
+      );
+      await Promise.all(batchPromises);
+      i += concurrency;
+    }
+    return results;
+  }
 
   var FetchState = /* @__PURE__ */ ((FetchState2) => {
     FetchState2[FetchState2["FAILED"] = 0] = "FAILED";
@@ -1352,11 +1379,9 @@ Report issues here: <a target="_blank" href="https://github.com/MapoMagpie/eh-vi
               [this.data, this.contentType] = ret;
               [this.data, this.contentType] = await this.matcher.processData(this.data, this.contentType, this.node.originSrc);
               if (this.contentType.startsWith("text")) {
-                if (this.data.byteLength < 1e5) {
-                  const str = new TextDecoder().decode(this.data);
-                  evLog("error", "unexpect content:\n", str);
-                  throw new Error(`expect image data, fetched wrong type: ${this.contentType}, the content is showing up in console(F12 open it).`);
-                }
+                const str = new TextDecoder().decode(this.data);
+                evLog("error", "unexpect content:\n", str);
+                throw new Error(`expect image data, fetched wrong type: ${this.contentType}, the content is showing up in console(F12 open it).`);
               }
               this.node.blobSrc = transient.imgSrcCSP ? this.node.originSrc : URL.createObjectURL(new Blob([this.data], { type: this.contentType }));
               this.node.mimeType = this.contentType;
@@ -3233,6 +3258,96 @@ Report issues here: <a target="_blank" href="https://github.com/MapoMagpie/eh-vi
     }
   }
 
+  class ArtStationMatcher extends BaseMatcher {
+    pageData = /* @__PURE__ */ new Map();
+    info = { username: "", projects: 0, assets: 0 };
+    tags = {};
+    name() {
+      return "Art Station";
+    }
+    galleryMeta() {
+      const meta = new GalleryMeta(window.location.href, `artstaion-${this.info.username}-w${this.info.projects}-p${this.info.assets}`);
+      meta.tags = this.tags;
+      return meta;
+    }
+    async *fetchPagesSource() {
+      const { id, username } = await this.fetchArtistInfo();
+      this.info.username = username;
+      let page = 0;
+      while (true) {
+        page++;
+        const projects = await this.fetchProjects(username, id.toString(), page);
+        if (!projects || projects.length === 0)
+          break;
+        this.pageData.set(page.toString(), projects);
+        yield page.toString();
+      }
+    }
+    async parseImgNodes(pageNo) {
+      const projects = this.pageData.get(pageNo);
+      if (!projects)
+        throw new Error("cannot get projects form page data");
+      const projectURLs = projects.map((p) => `https://www.artstation.com/projects/${p.hash_id}.json`);
+      const assets = await batchFetch(projectURLs, 10, "json");
+      let ret = [];
+      for (let asset of assets) {
+        this.info.projects++;
+        this.tags[asset.slug] = asset.tags;
+        for (let i = 0; i < asset.assets.length; i++) {
+          const a = asset.assets[i];
+          if (a.asset_type === "cover")
+            continue;
+          const thumb = a.image_url.replace("/large/", "/small/");
+          const ext = a.image_url.match(/\.(\w+)\?\d+$/)?.[1] ?? "jpg";
+          const title = `${asset.slug}-${i + 1}.${ext}`;
+          let originSrc = a.image_url;
+          if (a.has_embedded_player && a.player_embedded) {
+            if (a.player_embedded.includes("youtube"))
+              continue;
+            originSrc = a.player_embedded;
+          }
+          this.info.assets++;
+          ret.push(new ImageNode(thumb, asset.permalink, title, void 0, originSrc));
+        }
+      }
+      return ret;
+    }
+    async fetchOriginMeta(node) {
+      if (node.originSrc?.startsWith("<iframe")) {
+        const iframe = node.originSrc.match(/src=['"](.*?)['"]\s/)?.[1];
+        if (!iframe)
+          throw new Error("cannot match video clip url");
+        const doc = await window.fetch(iframe).then((res) => res.text()).then((text) => new DOMParser().parseFromString(text, "text/html"));
+        const source = doc.querySelector("video > source");
+        if (!source)
+          throw new Error("cannot find video element");
+        return { url: source.src };
+      }
+      return { url: node.originSrc };
+    }
+    async processData(data, contentType) {
+      if (contentType.startsWith("binary") || contentType.startsWith("text")) {
+        return [data, "video/mp4"];
+      }
+      return [data, contentType];
+    }
+    workURL() {
+      return /artstation.com\/[-\w]+(\/albums\/\d+)?$/;
+    }
+    async fetchArtistInfo() {
+      const user = window.location.pathname.slice(1).split("/").shift();
+      if (!user)
+        throw new Error("cannot match artist's username");
+      const info = await window.fetch(`https://www.artstation.com/users/${user}/quick.json`).then((res) => res.json());
+      return info;
+    }
+    async fetchProjects(user, id, page) {
+      const url = `https://www.artstation.com/users/${user}/projects.json?user_id=${id}&page=${page}`;
+      const project = await window.fetch(url).then((res) => res.json());
+      return project.data;
+    }
+  }
+
   class DanbooruMatcher extends BaseMatcher {
     tags = {};
     blacklistTags = [];
@@ -5423,10 +5538,10 @@ before contentType: ${contentType}, after contentType: ${blob.type}
         return list;
       const pidList = JSON.parse(source);
       this.fetchTagsByPids(pidList);
-      const pageListData = await fetchUrls(pidList.map((p) => `https://www.pixiv.net/ajax/illust/${p}/pages?lang=en`), 5);
+      const pageListData = await batchFetch(pidList.map((p) => `https://www.pixiv.net/ajax/illust/${p}/pages?lang=en`), 5, "json");
       for (let i = 0; i < pidList.length; i++) {
         const pid = pidList[i];
-        const data = JSON.parse(pageListData[i]);
+        const data = pageListData[i];
         if (data.error) {
           throw new Error(`Fetch page list error: ${data.message}`);
         }
@@ -5479,24 +5594,6 @@ before contentType: ${contentType}, after contentType: ${blob.type}
       }
     }
   }
-  async function fetchUrls(urls, concurrency) {
-    const results = new Array(urls.length);
-    let i = 0;
-    while (i < urls.length) {
-      const batch = urls.slice(i, i + concurrency);
-      const batchPromises = batch.map(
-        (url, index) => window.fetch(url).then((resp) => {
-          if (resp.ok) {
-            return resp.text();
-          }
-          throw new Error(`Failed to fetch ${url}: ${resp.status} ${resp.statusText}`);
-        }).then((raw) => results[index + i] = raw)
-      );
-      await Promise.all(batchPromises);
-      i += concurrency;
-    }
-    return results;
-  }
 
   class RokuHentaiMatcher extends BaseMatcher {
     name() {
@@ -5909,7 +6006,8 @@ before contentType: ${contentType}, after contentType: ${blob.type}
       new MHGMatcher(),
       new MangaCopyMatcher(),
       new E621Matcher(),
-      new ArcaMatcher()
+      new ArcaMatcher(),
+      new ArtStationMatcher()
     ];
   }
   function adaptMatcher(url) {

diff --git a/src/img-fetcher.ts b/src/img-fetcher.ts
@@ -119,11 +119,11 @@ export class IMGFetcher implements VisualNode {
             [this.data, this.contentType] = ret;
             [this.data, this.contentType] = await this.matcher.processData(this.data, this.contentType, this.node.originSrc!);
             if (this.contentType.startsWith("text")) {
-              if (this.data.byteLength < 100000) { // less then 100kb
-                const str = new TextDecoder().decode(this.data);
-                evLog("error", "unexpect content:\n", str);
-                throw new Error(`expect image data, fetched wrong type: ${this.contentType}, the content is showing up in console(F12 open it).`);
-              }
+              // if (this.data.byteLength < 100000) { // less then 100kb
+              const str = new TextDecoder().decode(this.data);
+              evLog("error", "unexpect content:\n", str);
+              throw new Error(`expect image data, fetched wrong type: ${this.contentType}, the content is showing up in console(F12 open it).`);
+              // }
             }
             this.node.blobSrc = transient.imgSrcCSP ? this.node.originSrc : URL.createObjectURL(new Blob([this.data], { type: this.contentType }));
             this.node.mimeType = this.contentType;

diff --git a/src/platform/adapt.ts b/src/platform/adapt.ts
@@ -1,6 +1,7 @@
 import { conf } from "../config";
 import { Comic18Matcher } from "./18comic";
 import { ArcaMatcher } from "./arca";
+import { ArtStationMatcher } from "./artstation";
 import { DanbooruDonmaiMatcher, E621Matcher, GelBooruMatcher, KonachanMatcher, Rule34Matcher, YandereMatcher } from "./danbooru";
 import { EHMatcher } from "./ehentai";
 import { HentaiNexusMatcher } from "./hentainexus";
@@ -41,6 +42,7 @@ export function getMatchers(): Matcher[] {
     new MangaCopyMatcher(),
     new E621Matcher(),
     new ArcaMatcher(),
+    new ArtStationMatcher(),
   ];
 }
 

diff --git a/src/platform/artstation.ts b/src/platform/artstation.ts
@@ -0,0 +1,133 @@
+import { GalleryMeta } from "../download/gallery-meta";
+import ImageNode from "../img-node";
+import { PagesSource } from "../page-fetcher";
+import { batchFetch } from "../utils/query";
+import { BaseMatcher, OriginMeta } from "./platform";
+
+export class ArtStationMatcher extends BaseMatcher {
+  pageData: Map<string, ArtStationProject[]> = new Map();
+  info: { username: string, projects: number, assets: number } = { username: "", projects: 0, assets: 0 };
+  tags: Record<string, string[]> = {};
+  name(): string {
+    return "Art Station";
+  }
+  galleryMeta(): GalleryMeta {
+    const meta = new GalleryMeta(window.location.href, `artstaion-${this.info.username}-w${this.info.projects}-p${this.info.assets}`);
+    meta.tags = this.tags;
+    return meta;
+  }
+  async *fetchPagesSource(): AsyncGenerator<PagesSource> {
+    // find artist id;
+    const { id, username } = await this.fetchArtistInfo();
+    this.info.username = username;
+    let page = 0;
+    while (true) {
+      page++;
+      const projects = await this.fetchProjects(username, id.toString(), page);
+      if (!projects || projects.length === 0) break;
+      this.pageData.set(page.toString(), projects);
+      yield page.toString();
+    }
+  }
+  async parseImgNodes(pageNo: PagesSource): Promise<ImageNode[]> {
+    const projects = this.pageData.get(pageNo as string);
+    if (!projects) throw new Error("cannot get projects form page data");
+    const projectURLs = projects.map(p => `https://www.artstation.com/projects/${p.hash_id}.json`)
+    const assets = await batchFetch<ArtStationAsset>(projectURLs, 10, "json");
+    let ret: ImageNode[] = [];
+    for (let asset of assets) {
+      this.info.projects++;
+      this.tags[asset.slug] = asset.tags;
+      for (let i = 0; i < asset.assets.length; i++) {
+        const a = asset.assets[i];
+        if (a.asset_type === "cover") continue;
+        const thumb = a.image_url.replace("/large/", "/small/");
+        const ext = a.image_url.match(/\.(\w+)\?\d+$/)?.[1] ?? "jpg";
+        const title = `${asset.slug}-${i + 1}.${ext}`;
+        let originSrc = a.image_url;
+        if (a.has_embedded_player && a.player_embedded) {
+          if (a.player_embedded.includes("youtube")) continue; // skip youtube embedded
+          originSrc = a.player_embedded;
+        }
+        this.info.assets++;
+        ret.push(new ImageNode(thumb, asset.permalink, title, undefined, originSrc));
+      }
+    }
+    return ret;
+  }
+  async fetchOriginMeta(node: ImageNode): Promise<OriginMeta> {
+    if (node.originSrc?.startsWith("<iframe")) {
+      const iframe = node.originSrc.match(/src=['"](.*?)['"]\s/)?.[1];
+      if (!iframe) throw new Error("cannot match video clip url");
+      const doc = await window.fetch(iframe).then(res => res.text()).then(text => new DOMParser().parseFromString(text, "text/html"));
+      const source = doc.querySelector<HTMLSourceElement>("video > source");
+      if (!source) throw new Error("cannot find video element");
+      return { url: source.src };
+    }
+    return { url: node.originSrc! };
+  }
+  async processData(data: Uint8Array, contentType: string): Promise<[Uint8Array, string]> {
+    if (contentType.startsWith("binary") || contentType.startsWith("text")) {
+      return [data, "video/mp4"];
+    }
+    return [data, contentType];
+  }
+  workURL(): RegExp {
+    return /artstation.com\/[-\w]+(\/albums\/\d+)?$/;
+  }
+  async fetchArtistInfo(): Promise<ArtStationArtistInfo> {
+    const user = window.location.pathname.slice(1).split("/").shift();
+    if (!user) throw new Error("cannot match artist's username");
+    const info = await window.fetch(`https://www.artstation.com/users/${user}/quick.json`).then(res => res.json()) as ArtStationArtistInfo;
+    return info;
+  }
+  async fetchProjects(user: string, id: string, page: number): Promise<ArtStationProject[]> {
+    const url = `https://www.artstation.com/users/${user}/projects.json?user_id=${id}&page=${page}`;
+    const project = await window.fetch(url).then(res => res.json()) as { data: ArtStationProject[], total_count: number };
+    return project.data;
+  }
+
+}
+
+type ArtStationArtistInfo = {
+  id: number,
+  full_name: string,
+  username: string,
+  permalink: string,
+}
+
+type ArtStationProject = {
+  id: number,
+  assets_count: number,
+  title: string,
+  description: string,
+  slug: string, // title
+  hash_id: string,
+  permalink: string, // href
+  cover: {
+    id: number,
+    small_square_url: string,
+    micro_square_image_url: string,
+    thumb_url: string
+  },
+}
+
+type ArtStationAsset = {
+  tags: string[],
+  assets: {
+    has_image: boolean,
+    has_embedded_player: boolean,
+    // "player_embedded": "<iframe src='https://www.artstation.com/api/v2/animation/video_clips/05283c5d-c7d9-496a-8906-73adabcbe407/embed.html?s=1ca70087a5cf3cb220bb3dc9a4818e63b88c741eeb9bfb89d1caac2832e7a353&t=1725866441' width='1920' height='2560' frameborder='0' allowfullscreen allows='autoplay; fullscreen' style='max-width: 1920px; max-height: 2560px;'></iframe>",
+    player_embedded?: string,
+    image_url: string,
+    width: number,
+    height: number,
+    position: number, // 0
+    asset_type: "image" | "cover" | "video_clip",
+
+  }[],
+  id: 19031208,
+  cover_url: string,
+  permalink: string,
+  slug: string,
+}