Merge pull request #7 from scrapfly/clob-blob

support clob and blob object handling

src/client.ts

@@ -92,6 +92,41 @@ export class ScrapflyClient {
     return new errors.ScrapflyError(message, args);
   }
 
+  /**
+   * Handle clob and blob large objects
+   */
+  async handleLargeObjects(result: any, format: "clob" | "blob"): Promise<ScrapeResult> {
+    let response: Response;
+
+    try {
+      const url = new URL(result.content);
+      const params = { key: this.key };
+      url.search = new URLSearchParams(params).toString();
+      response = await this.fetch({
+        url: url.toString(),
+        method: 'GET',
+        headers: {
+          'user-agent': this.ua,
+          'accept-encoding': 'gzip, deflate, br',
+          accept: 'application/json',
+        },
+      });
+    } catch (e) {
+      log.error('error', e);
+      throw e;
+    }
+
+    const content: string = await response.text();
+    result.content = content
+    if (format === 'clob') {
+      result.format = 'text'
+    }
+    if (format === 'blob') {
+      result.format = 'binary'
+    }
+    return result
+  }
+
   /**
    * Turn scrapfly API response to ScrapeResult or raise one of ScrapflyError
    */
@@ -173,6 +208,12 @@ export class ScrapflyClient {
       }
       throw new errors.ApiHttpClientError(JSON.stringify(data));
     }
+
+    const content_format = data.result.format
+    if (content_format === 'clob' || content_format === 'blob') {
+      data.result = await this.handleLargeObjects(data.result, content_format)
+    }
+
     const result = this.handleResponse(
       response,
       new ScrapeResult({