feat: add markdownify functionality

Author: DPende

scrapegraph-js/README.md

@@ -36,7 +36,6 @@ yarn add scrapegraph-js
 
 ```javascript
 import { smartScraper } from 'scrapegraph-js';
-import 'dotenv/config';
 
 // Initialize variables
 const apiKey = process.env.SGAI_APIKEY; // Set your API key as an environment variable
@@ -85,7 +84,6 @@ Here is a real-world example:
 ```javascript
 import { smartScraper } from 'scrapegraph-js';
 import { z } from 'zod';
-import 'dotenv/config';
 
 const apiKey = 'your-api-key';
 const url = 'https://scrapegraphai.com/';
@@ -107,6 +105,25 @@ const schema = z.object({
 })();
 ```
 
+### Markdownify
+Converts a webpage into clean, well-structured markdown format.
+```javascript
+import { smartScraper } from 'scrapegraph-js';
+
+const apiKey = "your_api_key";
+const url = 'https://scrapegraphai.com/';
+
+(async () => {
+  try {
+    const response = await markdownify(apiKey, url);
+    console.log(response);
+  } catch (error) {
+    console.error(error);
+  }
+})();
+```
+
+
 ### Checking API Credits
 
 ```javascript

scrapegraph-js/examples/markdownify_example.js

@@ -0,0 +1,35 @@
+import { getMarkdownifyRequest, markdownify } from 'scrapegraph-js';
+import fs from 'fs';
+import 'dotenv/config';
+
+// markdownify function example
+const apiKey = process.env.SGAI_APIKEY;
+const url = 'https://scrapegraphai.com/';
+
+try {
+  const response = await markdownify(apiKey, url);
+  console.log(response);
+  saveFile(response.result);
+} catch (error) {
+  console.error(error);
+}
+
+// helper function for save the file locally
+function saveFile(output) {
+  try {
+    fs.writeFileSync('result.md', output);
+    console.log('Success!');
+  } catch (err) {
+    console.error('Error during the file writing: ', err);
+  }
+}
+
+// getMarkdownifyRequest function example
+const requestId = '2563b972-cb6f-400b-be76-edb235458560';
+
+try {
+  const response = await getMarkdownifyRequest(apiKey, requestId);
+  console.log(response);
+} catch (error) {
+  console.log(error);
+}

scrapegraph-js/index.js

@@ -1,3 +1,4 @@
 export { smartScraper, getSmartScraperRequest } from './src/smartScraper.js';
+export { markdownify, getMarkdownifyRequest } from './src/markdownify.js';
 export { getCredits } from './src/credits.js';
-export { sendFeedback } from './src/feedback.js'; 
+export { sendFeedback } from './src/feedback.js';

scrapegraph-js/src/markdownify.js

@@ -0,0 +1,52 @@
+import axios from 'axios';
+import handleError from './utils/handleError.js';
+
+/**
+ * Converts a webpage into clean, well-structured markdown format.
+ *
+ * @param {string} apiKey - Your ScrapeGraph AI API key.
+ * @param {string} url - The URL of the webpage to be converted.
+ * @returns {Promise<string>} A promise that resolves to the markdown representation of the webpage.
+ * @throws {Error} Throws an error if the HTTP request fails.
+ */
+export async function markdownify(apiKey, url){
+  const endpoint = 'https://api.scrapegraphai.com/v1/markdownify';
+  const headers = {
+    'accept': 'application/json',
+    'SGAI-APIKEY': apiKey,
+  };
+
+  const payload = {
+    website_url: url,
+  };
+
+  try {
+    const response = await axios.post(endpoint, payload, { headers });
+    return response.data;
+  } catch (error) {
+    handleError(error)
+  }
+}
+
+/**
+ * Retrieves the status or result of a markdownify request, with the option to review results from previous requests.
+ *
+ * @param {string} apiKey - Your ScrapeGraph AI API key.
+ * @param {string} requestId - The unique identifier for the markdownify request whose result you want to retrieve.
+ * @returns {Promise<string>} A promise that resolves with details about the status or outcome of the specified request.
+ * @throws {Error} Throws an error if the HTTP request fails.
+ */
+export async function getMarkdownifyRequest(apiKey, requestId){
+  const endpoint = 'https://api.scrapegraphai.com/v1/markdownify/' + requestId;
+  const headers = {
+    'accept': 'application/json',
+    'SGAI-APIKEY': apiKey,
+  };
+
+  try {
+    const response = await axios.get(endpoint, { headers });
+    return response.data;
+  } catch (error) {
+    handleError(error)
+  }
+}