Audit the Open Graph tags of every page in your sitemap

import Sitemapper from 'sitemapper';
import mql from '@microlink/mql';
import pMap from 'p-map';
import fs from  'fs'

// CONFIGURATION
const SITEMAP_URL = process.env.SITEMAP_URL || 'https://YOUR_WEB_PAGE.com/sitemap.xml';
const API_KEY = process.env.API_KEY; // Optional, but recommended to large sitemaps

const CONCURRENCY = 1; // Keep it low for free tier
const FREE_TIER_LIMIT = 25; // 25 requests per day for free tier
const FIRST_BATCH = 0; // If the free tier gets small make batch by batch

const validateUrl = async (url) => {
  await new Promise(resolve => setTimeout(resolve, 1000)); // Avoid free tier limits
  try {
    // We request 'meta' data specifically.
    // Microlink mimics a browser to extract OG tags.
    const { status, data, response } = await mql(url, {
      meta: true,
      apiKey: API_KEY
    });

    if (status !== 'success') {
      return { url, error: 'API Error', status };
    }

    const errors = [];
    
    // Validation logic

    if (!data.image || !data.image.url) errors.push('Missing OG Image');
    
    if (!data.title) errors.push('Missing OG Title');
    else if (data.title.length > 60) errors.push('OG Title is too long')

    if (!data.description) errors.push('Missing Description');
    else if (data.description.length < 50) errors.push('Description too short');

    if (!data.author) errors.push('Missing Author');

    if (!data.date) errors.push('Missing Date');
    else if (isNaN(new Date(data.date).getTime())) errors.push('Invalid Date');

    if (!data.logo) errors.push('Missing Logo');

    return { valid: !errors.length, url, errors };

  } catch (err) {
    return { url, error: err.message, valid: false };
  }
};

const runAudit = async () => {
  console.log(`🗺️  Fetching sitemap: ${SITEMAP_URL}...`);
  
  const sitemap = new Sitemapper({ url: SITEMAP_URL, timeout: 15000 });

  let { sites } = await sitemap.fetch();
  if (sites.length > FREE_TIER_LIMIT && !API_KEY) {
    console.log(`Total URLs exceeds free tier limit. Cutting from ${sites.length} to ${FREE_TIER_LIMIT}` );
    sites = sites.slice(FREE_TIER_LIMIT * FIRST_BATCH, FREE_TIER_LIMIT);
  }

  const totalUrls = sites.length;
  let processedCount = 0;
  
  console.log(`Found ${totalUrls} URLs. Starting validation...`);

  // Use p-map to control concurrency with progress tracking
  const results = await pMap(sites, async (url) => {
    const result = await validateUrl(url);
    processedCount++;
    const percent = ((processedCount / totalUrls) * 100).toFixed(1);
    process.stdout.write(`\r⏳ Progress: ${processedCount}/${totalUrls} (${percent}%)`);
    return result;
  }, { concurrency: CONCURRENCY });

  // Reporting
  const failures = results.filter(r => !r.valid);
  
  console.log('\n\n--- 📊 AUDIT REPORT ---');
  console.log(`Total Scanned: ${results.length}`);
  console.log(`Passed: ${results.length - failures.length}`);
  console.log(`Failed: ${failures.length}`);
  
  if (failures.length > 0) {
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const errorFilePath = `./errors-${timestamp}.txt`;
    
    let errorContent = '❌ FAILED URLS:\n';
    failures.forEach(f => {
      errorContent += `\n${f.url}\n`;
      if (f.errors) f.errors.forEach(e => errorContent += `   - ${e}\n`);
      if (f.error) errorContent += `   - System Error: ${f.error}\n`;
    });

    fs.writeFileSync(errorFilePath, errorContent);
    console.log(`\n❌ Errors saved to: ${errorFilePath}`);
  } else {
    console.log('\n✅ All systems nominal. Your sitemap is perfect.');
  }
};

runAudit();

const mql = require('@microlink/mql')

// ... prev code
const { status, data, response } = await mql(url, {
  meta: true,
  data: {
    content: {
      selector: 'body', // get the body of the page
      type: 'text' // get combined text content
    }
  }
})

console.log(`The content of the url -> ${data.content}`)
// next code ...

if (failures.length > 0) {
  process.exit(1);
}