robots.txt and Sitemap.xml: SEO Configuration Analysis for Developers

Published: January 2024
Reading time: 9 minutes

Every website broadcasts its structure to search engines through robots.txt and sitemap.xml. These files reveal crawling permissions, hidden directories, and content priorities. Here's how to analyze them for SEO insights, security issues, and optimization opportunities.

Why These Files Matter

The SEO Impact

• Crawl efficiency - Direct bots to important content
• Index control - Prevent duplicate content issues
• Discovery speed - New pages found faster
• Crawl budget - Don't waste it on junk pages
• Security leaks - robots.txt exposes hidden paths

Real-World Issues

Major retailer: Blocked entire site with robots.txt typo
News site: 90% of articles missing from sitemap
SaaS company: Exposed admin URLs in robots.txt
E-commerce: Blocked product images, lost rich snippets
Government site: Sensitive directories listed in robots.txt

Quick robots.txt Analysis

1. robots.txt Parser and Analyzer

// Analyze robots.txt for any website
async function analyzeRobotsTxt(domain = window.location.hostname) {
  console.log(`🤖 robots.txt Analysis for ${domain}\n`);
  
  const robotsUrl = `${window.location.protocol}//${domain}/robots.txt`;
  
  try {
    const response = await fetch(robotsUrl);
    
    if (!response.ok) {
      console.log(`❌ No robots.txt found (${response.status})`);
      console.log('   Impact: Search engines will crawl everything');
      console.log('   Recommendation: Create a robots.txt file');
      return;
    }
    
    const robotsTxt = await response.text();
    console.log(`✅ robots.txt found (${robotsTxt.length} bytes)\n`);
    
    // Parse robots.txt
    const rules = parseRobotsTxt(robotsTxt);
    
    // Analyze rules
    analyzeRules(rules);
    
    // Security analysis
    analyzeSecurityImplications(rules);
    
    // SEO analysis
    analyzeSEOImplications(rules);
    
    // Common mistakes
    detectCommonMistakes(robotsTxt, rules);
    
  } catch (error) {
    console.log('❌ Error fetching robots.txt:', error.message);
  }
}

function parseRobotsTxt(content) {
  const rules = {
    userAgents: new Map(),
    sitemaps: [],
    crawlDelay: null,
    host: null
  };
  
  let currentAgent = '*';
  const lines = content.split('\n');
  
  lines.forEach(line => {
    line = line.trim();
    
    // Skip comments and empty lines
    if (!line || line.startsWith('#')) return;
    
    const [directive, ...valueParts] = line.split(':');
    const value = valueParts.join(':').trim();
    
    switch(directive.toLowerCase()) {
      case 'user-agent':
        currentAgent = value.toLowerCase();
        if (!rules.userAgents.has(currentAgent)) {
          rules.userAgents.set(currentAgent, {
            allow: [],
            disallow: [],
            crawlDelay: null
          });
        }
        break;
        
      case 'allow':
        if (!rules.userAgents.has(currentAgent)) {
          rules.userAgents.set(currentAgent, { allow: [], disallow: [] });
        }
        rules.userAgents.get(currentAgent).allow.push(value);
        break;
        
      case 'disallow':
        if (!rules.userAgents.has(currentAgent)) {
          rules.userAgents.set(currentAgent, { allow: [], disallow: [] });
        }
        rules.userAgents.get(currentAgent).disallow.push(value);
        break;
        
      case 'crawl-delay':
        if (rules.userAgents.has(currentAgent)) {
          rules.userAgents.get(currentAgent).crawlDelay = parseInt(value);
        }
        rules.crawlDelay = parseInt(value);
        break;
        
      case 'sitemap':
        rules.sitemaps.push(value);
        break;
        
      case 'host':
        rules.host = value;
        break;
    }
  });
  
  return rules;
}

function analyzeRules(rules) {
  console.log('📋 Parsed Rules:\n');
  
  // User agents
  console.log(`User Agents: ${rules.userAgents.size}`);
  rules.userAgents.forEach((agentRules, agent) => {
    console.log(`\n🤖 ${agent}:`);
    
    if (agentRules.disallow.length > 0) {
      console.log('  ❌ Disallowed:');
      agentRules.disallow.slice(0, 5).forEach(path => {
        console.log(`     ${path}`);
      });
      if (agentRules.disallow.length > 5) {
        console.log(`     ... and ${agentRules.disallow.length - 5} more`);
      }
    }
    
    if (agentRules.allow.length > 0) {
      console.log('  ✅ Allowed:');
      agentRules.allow.slice(0, 5).forEach(path => {
        console.log(`     ${path}`);
      });
    }
    
    if (agentRules.crawlDelay) {
      console.log(`  ⏱️ Crawl delay: ${agentRules.crawlDelay}s`);
    }
  });
  
  // Sitemaps
  if (rules.sitemaps.length > 0) {
    console.log('\n🗺️ Sitemaps:');
    rules.sitemaps.forEach(sitemap => {
      console.log(`  ${sitemap}`);
    });
  } else {
    console.log('\n⚠️ No sitemap declared in robots.txt');
  }
}

function analyzeSecurityImplications(rules) {
  console.log('\n🔒 Security Analysis:\n');
  
  const sensitivePatterns = [
    { pattern: /admin/i, type: 'Admin panel' },
    { pattern: /api/i, type: 'API endpoints' },
    { pattern: /backup/i, type: 'Backup files' },
    { pattern: /config/i, type: 'Configuration' },
    { pattern: /database/i, type: 'Database' },
    { pattern: /private/i, type: 'Private content' },
    { pattern: /staging/i, type: 'Staging environment' },
    { pattern: /test/i, type: 'Test environment' },
    { pattern: /\.git/i, type: 'Git repository' },
    { pattern: /\.env/i, type: 'Environment files' },
    { pattern: /wp-admin/i, type: 'WordPress admin' },
    { pattern: /phpmyadmin/i, type: 'phpMyAdmin' }
  ];
  
  const exposedPaths = [];
  
  rules.userAgents.forEach((agentRules, agent) => {
    agentRules.disallow.forEach(path => {
      sensitivePatterns.forEach(({ pattern, type }) => {
        if (pattern.test(path)) {
          exposedPaths.push({ path, type, agent });
        }
      });
    });
  });
  
  if (exposedPaths.length > 0) {
    console.log('⚠️ Sensitive paths exposed in robots.txt:');
    exposedPaths.forEach(({ path, type }) => {
      console.log(`  ${type}: ${path}`);
    });
    console.log('\n  📌 Consider: These paths are now public knowledge!');
    console.log('  💡 Better approach: Use authentication, not robots.txt');
  } else {
    console.log('✅ No obvious sensitive paths exposed');
  }
  
  // Check for security through obscurity
  const obscurityPaths = rules.userAgents.get('*')?.disallow.filter(path => 
    path.includes('secret') || path.includes('hidden') || path.includes('private')
  );
  
  if (obscurityPaths?.length > 0) {
    console.log('\n❌ Security through obscurity detected:');
    obscurityPaths.forEach(path => {
      console.log(`  ${path}`);
    });
  }
}

function analyzeSEOImplications(rules) {
  console.log('\n📈 SEO Analysis:\n');
  
  const allRules = rules.userAgents.get('*') || { allow: [], disallow: [] };
  const googleRules = rules.userAgents.get('googlebot') || allRules;
  
  // Check if entire site is blocked
  if (allRules.disallow.includes('/')) {
    console.log('🚨 CRITICAL: Entire site blocked from all crawlers!');
    return;
  }
  
  // Check important paths
  const importantPaths = [
    { path: '/sitemap', importance: 'Sitemap access' },
    { path: '/*.js$', importance: 'JavaScript files' },
    { path: '/*.css$', importance: 'CSS files' },
    { path: '/images/', importance: 'Image directory' },
    { path: '/*?', importance: 'URL parameters' }
  ];
  
  console.log('Important paths status:');
  importantPaths.forEach(({ path, importance }) => {
    const isBlocked = googleRules.disallow.some(rule => 
      path.includes(rule) || rule.includes(path)
    );
    console.log(`  ${isBlocked ? '❌' : '✅'} ${importance} - ${isBlocked ? 'Blocked' : 'Allowed'}`);
  });
  
  // Crawl delay analysis
  if (rules.crawlDelay) {
    console.log(`\n⏱️ Crawl delay: ${rules.crawlDelay} seconds`);
    if (rules.crawlDelay > 10) {
      console.log('  ⚠️ Very high crawl delay may slow indexing');
    }
  }
  
  // Check for duplicate content issues
  const parameterBlocking = allRules.disallow.some(rule => rule.includes('?'));
  if (!parameterBlocking) {
    console.log('\n⚠️ URL parameters not blocked - possible duplicate content');
  }
}

function detectCommonMistakes(content, rules) {
  console.log('\n⚠️ Common Mistakes Check:\n');
  
  const issues = [];
  
  // Check for typos
  if (content.includes('Dissallow')) {
    issues.push('Typo: "Dissallow" should be "Disallow"');
  }
  
  if (content.includes('User-Agent')) {
    issues.push('Case sensitivity: "User-Agent" should be "User-agent"');
  }
  
  // Check for wildcards in User-agent
  rules.userAgents.forEach((_, agent) => {
    if (agent.includes('*') && agent !== '*') {
      issues.push(`Invalid wildcard in User-agent: ${agent}`);
    }
  });
  
  // Check for missing trailing slashes
  rules.userAgents.forEach(agentRules => {
    agentRules.disallow.forEach(path => {
      if (path.length > 1 && !path.endsWith('/') && !path.includes('.') && !path.includes('*')) {
        issues.push(`Missing trailing slash: ${path} (will also block ${path}*)`);
      }
    });
  });
  
  // Check for contradictions
  rules.userAgents.forEach(agentRules => {
    agentRules.disallow.forEach(disallowPath => {
      agentRules.allow.forEach(allowPath => {
        if (allowPath.startsWith(disallowPath)) {
          issues.push(`Contradiction: ${allowPath} allowed but parent ${disallowPath} is blocked`);
        }
      });
    });
  });
  
  if (issues.length > 0) {
    issues.forEach(issue => {
      console.log(`❌ ${issue}`);
    });
  } else {
    console.log('✅ No common mistakes detected');
  }
  
  return issues;
}

// Analyze current site
analyzeRobotsTxt();

2. Sitemap.xml Analyzer

// Analyze sitemap.xml for SEO insights
async function analyzeSitemap(domain = window.location.hostname) {
  console.log(`\n🗺️ Sitemap.xml Analysis for ${domain}\n`);
  
  // Common sitemap locations
  const sitemapUrls = [
    `${window.location.protocol}//${domain}/sitemap.xml`,
    `${window.location.protocol}//${domain}/sitemap_index.xml`,
    `${window.location.protocol}//${domain}/sitemaps/sitemap.xml`,
    `${window.location.protocol}//${domain}/sitemap`
  ];
  
  let sitemapFound = false;
  let sitemapUrl = null;
  let sitemapContent = null;
  
  // Try to find sitemap
  for (const url of sitemapUrls) {
    try {
      const response = await fetch(url);
      if (response.ok) {
        sitemapFound = true;
        sitemapUrl = url;
        sitemapContent = await response.text();
        break;
      }
    } catch (e) {
      // Continue trying
    }
  }
  
  if (!sitemapFound) {
    console.log('❌ No sitemap.xml found');
    console.log('   Tried locations:');
    sitemapUrls.forEach(url => console.log(`   - ${url}`));
    console.log('\n   Impact: Slower content discovery by search engines');
    console.log('   Recommendation: Create a sitemap.xml');
    return;
  }
  
  console.log(`✅ Sitemap found: ${sitemapUrl}`);
  console.log(`   Size: ${(sitemapContent.length / 1024).toFixed(1)}KB\n`);
  
  // Parse sitemap
  const sitemap = parseSitemap(sitemapContent);
  
  // Analyze sitemap
  analyzeSitemapContent(sitemap);
  
  // Check for issues
  detectSitemapIssues(sitemap);
  
  // SEO recommendations
  provideSitemapRecommendations(sitemap);
}

function parseSitemap(xmlContent) {
  const parser = new DOMParser();
  const doc = parser.parseFromString(xmlContent, 'text/xml');
  
  const sitemap = {
    type: 'urlset',
    urls: [],
    sitemaps: [],
    errors: []
  };
  
  // Check for parsing errors
  const parserError = doc.querySelector('parsererror');
  if (parserError) {
    sitemap.errors.push('XML parsing error: ' + parserError.textContent);
    return sitemap;
  }
  
  // Check if it's a sitemap index
  const sitemapIndex = doc.querySelector('sitemapindex');
  if (sitemapIndex) {
    sitemap.type = 'sitemapindex';
    const sitemaps = doc.querySelectorAll('sitemap');
    sitemaps.forEach(sm => {
      sitemap.sitemaps.push({
        loc: sm.querySelector('loc')?.textContent,
        lastmod: sm.querySelector('lastmod')?.textContent
      });
    });
    return sitemap;
  }
  
  // Parse regular sitemap
  const urls = doc.querySelectorAll('url');
  urls.forEach(url => {
    const urlEntry = {
      loc: url.querySelector('loc')?.textContent,
      lastmod: url.querySelector('lastmod')?.textContent,
      changefreq: url.querySelector('changefreq')?.textContent,
      priority: parseFloat(url.querySelector('priority')?.textContent || '0.5')
    };
    
    // Check for image extensions
    const images = url.querySelectorAll('image\\:image');
    if (images.length > 0) {
      urlEntry.images = Array.from(images).map(img => ({
        loc: img.querySelector('image\\:loc')?.textContent,
        title: img.querySelector('image\\:title')?.textContent,
        caption: img.querySelector('image\\:caption')?.textContent
      }));
    }
    
    // Check for video extensions
    const videos = url.querySelectorAll('video\\:video');
    if (videos.length > 0) {
      urlEntry.videos = Array.from(videos).map(vid => ({
        thumbnail: vid.querySelector('video\\:thumbnail_loc')?.textContent,
        title: vid.querySelector('video\\:title')?.textContent,
        description: vid.querySelector('video\\:description')?.textContent
      }));
    }
    
    sitemap.urls.push(urlEntry);
  });
  
  return sitemap;
}

function analyzeSitemapContent(sitemap) {
  console.log('📊 Sitemap Analysis:\n');
  
  if (sitemap.type === 'sitemapindex') {
    console.log(`Type: Sitemap Index`);
    console.log(`Sub-sitemaps: ${sitemap.sitemaps.length}`);
    
    sitemap.sitemaps.forEach((sm, index) => {
      console.log(`\n  ${index + 1}. ${sm.loc}`);
      if (sm.lastmod) {
        const days = Math.floor((Date.now() - new Date(sm.lastmod)) / (1000 * 60 * 60 * 24));
        console.log(`     Last modified: ${days} days ago`);
      }
    });
    
    return;
  }
  
  console.log(`Total URLs: ${sitemap.urls.length}`);
  
  // Analyze URL patterns
  const urlPatterns = {
    homePage: 0,
    withTrailingSlash: 0,
    withoutTrailingSlash: 0,
    withParameters: 0,
    withFragment: 0,
    https: 0,
    http: 0
  };
  
  sitemap.urls.forEach(url => {
    if (url.loc.endsWith('/') && url.loc.split('/').length === 4) urlPatterns.homePage++;
    if (url.loc.endsWith('/')) urlPatterns.withTrailingSlash++;
    else urlPatterns.withoutTrailingSlash++;
    if (url.loc.includes('?')) urlPatterns.withParameters++;
    if (url.loc.includes('#')) urlPatterns.withFragment++;
    if (url.loc.startsWith('https://')) urlPatterns.https++;
    if (url.loc.startsWith('http://')) urlPatterns.http++;
  });
  
  console.log('\nURL Patterns:');
  console.log(`  HTTPS: ${urlPatterns.https}`);
  console.log(`  HTTP: ${urlPatterns.http}`);
  console.log(`  With parameters: ${urlPatterns.withParameters}`);
  console.log(`  With fragments: ${urlPatterns.withFragment}`);
  
  // Analyze priorities
  const priorities = {};
  sitemap.urls.forEach(url => {
    const priority = url.priority.toFixed(1);
    priorities[priority] = (priorities[priority] || 0) + 1;
  });
  
  console.log('\nPriority Distribution:');
  Object.entries(priorities)
    .sort((a, b) => parseFloat(b[0]) - parseFloat(a[0]))
    .forEach(([priority, count]) => {
      console.log(`  ${priority}: ${count} URLs`);
    });
  
  // Analyze change frequencies
  const changeFreqs = {};
  sitemap.urls.forEach(url => {
    if (url.changefreq) {
      changeFreqs[url.changefreq] = (changeFreqs[url.changefreq] || 0) + 1;
    }
  });
  
  if (Object.keys(changeFreqs).length > 0) {
    console.log('\nChange Frequencies:');
    Object.entries(changeFreqs).forEach(([freq, count]) => {
      console.log(`  ${freq}: ${count} URLs`);
    });
  }
  
  // Analyze lastmod dates
  const lastmods = sitemap.urls.filter(url => url.lastmod).map(url => new Date(url.lastmod));
  if (lastmods.length > 0) {
    const newest = new Date(Math.max(...lastmods));
    const oldest = new Date(Math.min(...lastmods));
    const daysOld = Math.floor((Date.now() - oldest) / (1000 * 60 * 60 * 24));
    const daysNew = Math.floor((Date.now() - newest) / (1000 * 60 * 60 * 24));
    
    console.log('\nLast Modified Dates:');
    console.log(`  Newest: ${daysNew} days ago`);
    console.log(`  Oldest: ${daysOld} days ago`);
    console.log(`  URLs with lastmod: ${lastmods.length}/${sitemap.urls.length}`);
  }
  
  // Check for media
  const withImages = sitemap.urls.filter(url => url.images?.length > 0).length;
  const withVideos = sitemap.urls.filter(url => url.videos?.length > 0).length;
  
  if (withImages > 0 || withVideos > 0) {
    console.log('\nMedia Content:');
    if (withImages > 0) console.log(`  URLs with images: ${withImages}`);
    if (withVideos > 0) console.log(`  URLs with videos: ${withVideos}`);
  }
}

function detectSitemapIssues(sitemap) {
  console.log('\n⚠️ Issue Detection:\n');
  
  const issues = [];
  
  // Check sitemap size
  if (sitemap.urls.length > 50000) {
    issues.push({
      severity: 'high',
      issue: `Sitemap has ${sitemap.urls.length} URLs (max: 50,000)`,
      fix: 'Split into multiple sitemaps using a sitemap index'
    });
  }
  
  // Check for duplicates
  const urlSet = new Set();
  const duplicates = [];
  sitemap.urls.forEach(url => {
    if (urlSet.has(url.loc)) {
      duplicates.push(url.loc);
    }
    urlSet.add(url.loc);
  });
  
  if (duplicates.length > 0) {
    issues.push({
      severity: 'medium',
      issue: `${duplicates.length} duplicate URLs found`,
      fix: 'Remove duplicate entries'
    });
  }
  
  // Check for non-canonical URLs
  const nonCanonical = sitemap.urls.filter(url => 
    url.loc.includes('?') || 
    url.loc.includes('#') ||
    url.loc.includes('//')
  );
  
  if (nonCanonical.length > 0) {
    issues.push({
      severity: 'medium',
      issue: `${nonCanonical.length} non-canonical URLs (with ?, #, or //))`,
      fix: 'Use canonical URLs only'
    });
  }
  
  // Check for mixed protocols
  const protocols = new Set(sitemap.urls.map(url => url.loc.split(':')[0]));
  if (protocols.size > 1) {
    issues.push({
      severity: 'high',
      issue: 'Mixed protocols (HTTP and HTTPS)',
      fix: 'Use consistent protocol (preferably HTTPS)'
    });
  }
  
  // Check for missing lastmod
  const withoutLastmod = sitemap.urls.filter(url => !url.lastmod).length;
  if (withoutLastmod > sitemap.urls.length * 0.5) {
    issues.push({
      severity: 'low',
      issue: `${withoutLastmod} URLs without lastmod date`,
      fix: 'Add lastmod to help search engines prioritize crawling'
    });
  }
  
  // Display issues
  if (issues.length > 0) {
    issues.forEach(({ severity, issue, fix }) => {
      const icon = severity === 'high' ? '🔴' : 
                  severity === 'medium' ? '🟡' : '🟢';
      console.log(`${icon} ${issue}`);
      console.log(`   Fix: ${fix}`);
    });
  } else {
    console.log('✅ No major issues detected');
  }
  
  return issues;
}

function provideSitemapRecommendations(sitemap) {
  console.log('\n💡 SEO Recommendations:\n');
  
  const recommendations = [];
  
  // Check if sitemap is too small
  if (sitemap.urls.length < 10) {
    recommendations.push('Consider adding more pages to your sitemap');
  }
  
  // Check for homepage
  const hasHomepage = sitemap.urls.some(url => {
    const path = new URL(url.loc).pathname;
    return path === '/' || path === '';
  });
  
  if (!hasHomepage) {
    recommendations.push('Add your homepage to the sitemap');
  }
  
  // Check priority usage
  const uniquePriorities = new Set(sitemap.urls.map(url => url.priority));
  if (uniquePriorities.size === 1) {
    recommendations.push('Use varied priorities to indicate page importance');
  }
  
  // Check for images
  const hasImages = sitemap.urls.some(url => url.images?.length > 0);
  if (!hasImages) {
    recommendations.push('Consider adding image sitemap extensions for better image SEO');
  }
  
  // Check update frequency
  const lastmods = sitemap.urls.filter(url => url.lastmod);
  if (lastmods.length > 0) {
    const avgAge = lastmods.reduce((sum, url) => {
      return sum + (Date.now() - new Date(url.lastmod));
    }, 0) / lastmods.length;
    
    const avgDays = avgAge / (1000 * 60 * 60 * 24);
    if (avgDays > 180) {
      recommendations.push('Many pages haven\'t been updated in 6+ months');
    }
  }
  
  if (recommendations.length > 0) {
    recommendations.forEach(rec => {
      console.log(`• ${rec}`);
    });
  } else {
    console.log('✅ Sitemap follows best practices');
  }
  
  // Additional tips
  console.log('\n📌 Best Practices:');
  console.log('• Submit sitemap to Google Search Console');
  console.log('• Include sitemap URL in robots.txt');
  console.log('• Update sitemap when content changes');
  console.log('• Monitor crawl stats in search console');
  console.log('• Use separate sitemaps for different content types');
}

// Analyze current site's sitemap
analyzeSitemap();

3. Crawlability Tester

// Test if important pages are crawlable
async function testCrawlability() {
  console.log('\n🕷️ Crawlability Test\n');
  
  const testPaths = [
    { path: '/', type: 'Homepage' },
    { path: '/robots.txt', type: 'Robots file' },
    { path: '/sitemap.xml', type: 'Sitemap' },
    { path: '/favicon.ico', type: 'Favicon' },
    { path: '/wp-admin/', type: 'WordPress admin' },
    { path: '/.git/', type: 'Git directory' },
    { path: '/.env', type: 'Environment file' },
    { path: '/api/', type: 'API directory' },
    { path: '/admin/', type: 'Admin panel' },
    { path: '/search', type: 'Search page' }
  ];
  
  // First, get robots.txt rules
  let robotsRules = null;
  try {
    const robotsResponse = await fetch('/robots.txt');
    if (robotsResponse.ok) {
      const robotsTxt = await robotsResponse.text();
      robotsRules = parseRobotsTxt(robotsTxt);
    }
  } catch (e) {
    console.log('Could not fetch robots.txt');
  }
  
  console.log('Testing crawl access for common paths:\n');
  
  for (const { path, type } of testPaths) {
    const url = `${window.location.origin}${path}`;
    
    // Check robots.txt rules
    let robotsAllowed = true;
    if (robotsRules) {
      const rules = robotsRules.userAgents.get('*') || { disallow: [] };
      robotsAllowed = !rules.disallow.some(rule => {
        if (rule === path) return true;
        if (rule.endsWith('/') && path.startsWith(rule)) return true;
        return false;
      });
    }
    
    // Check actual accessibility
    let accessible = false;
    let status = null;
    
    try {
      const response = await fetch(url, { 
        method: 'HEAD',
        mode: 'no-cors' // Avoid CORS issues
      });
      status = response.status;
      accessible = response.ok || response.status === 403; // 403 means it exists but is protected
    } catch (e) {
      accessible = false;
    }
    
    // Determine crawlability
    const icon = accessible && robotsAllowed ? '✅' : 
                accessible && !robotsAllowed ? '🚫' :
                !accessible && robotsAllowed ? '❌' : '❌';
    
    const crawlStatus = accessible && robotsAllowed ? 'Crawlable' :
                       accessible && !robotsAllowed ? 'Blocked by robots.txt' :
                       !accessible ? 'Not accessible' : 'Blocked';
    
    console.log(`${icon} ${type.padEnd(20)} ${crawlStatus}`);
    
    // Security warnings
    if (type.includes('admin') && accessible && robotsAllowed) {
      console.log('   ⚠️ Admin area is publicly crawlable!');
    }
    if (type.includes('Git') && accessible) {
      console.log('   🚨 Git directory exposed!');
    }
    if (type.includes('Environment') && accessible) {
      console.log('   🚨 Environment file exposed!');
    }
  }
  
  // Test meta robots
  console.log('\n🏷️ Meta Robots Analysis:\n');
  
  const metaRobots = document.querySelector('meta[name="robots"]');
  if (metaRobots) {
    const content = metaRobots.getAttribute('content');
    console.log(`Meta robots: ${content}`);
    
    if (content.includes('noindex')) {
      console.log('  ⚠️ Page is set to noindex!');
    }
    if (content.includes('nofollow')) {
      console.log('  ⚠️ Links are set to nofollow!');
    }
  } else {
    console.log('No meta robots tag (page is indexable)');
  }
  
  // Check X-Robots-Tag header
  try {
    const response = await fetch(window.location.href, { method: 'HEAD' });
    const xRobots = response.headers.get('x-robots-tag');
    if (xRobots) {
      console.log(`\nX-Robots-Tag header: ${xRobots}`);
    }
  } catch (e) {
    // Ignore
  }
}

testCrawlability();

Complete SEO Configuration Audit

Comprehensive SEO File Analysis

// Run complete SEO configuration audit
async function completeSEOAudit() {
  console.log('🔍 COMPLETE SEO CONFIGURATION AUDIT');
  console.log('═'.repeat(50));
  console.log(`Domain: ${window.location.hostname}`);
  console.log(`Date: ${new Date().toLocaleDateString()}\n`);
  
  const audit = {
    score: 100,
    robots: { exists: false, issues: [] },
    sitemap: { exists: false, issues: [] },
    crawlability: { accessible: 0, blocked: 0 },
    security: { exposed: [] },
    recommendations: []
  };
  
  // Phase 1: robots.txt Analysis
  console.log('🤖 PHASE 1: ROBOTS.TXT ANALYSIS');
  console.log('─'.repeat(40));
  
  try {
    const robotsResponse = await fetch('/robots.txt');
    audit.robots.exists = robotsResponse.ok;
    
    if (robotsResponse.ok) {
      console.log('✅ robots.txt found');
      const robotsTxt = await robotsResponse.text();
      const rules = parseRobotsTxt(robotsTxt);
      
      // Check for issues
      if (rules.userAgents.get('*')?.disallow.includes('/')) {
        audit.robots.issues.push('Entire site blocked');
        audit.score -= 50;
      }
      
      // Check for exposed sensitive paths
      const sensitivePaths = ['/admin', '/api', '/private', '/.git'];
      rules.userAgents.forEach(agent => {
        agent.disallow.forEach(path => {
          if (sensitivePaths.some(sp => path.includes(sp))) {
            audit.security.exposed.push(path);
          }
        });
      });
      
      if (audit.security.exposed.length > 0) {
        audit.score -= 10;
      }
      
      // Check for sitemap declaration
      if (rules.sitemaps.length === 0) {
        audit.robots.issues.push('No sitemap declared');
        audit.score -= 5;
      }
      
    } else {
      console.log('❌ No robots.txt found');
      audit.recommendations.push('Create robots.txt file');
      audit.score -= 10;
    }
  } catch (e) {
    console.log('Error checking robots.txt');
  }
  
  // Phase 2: Sitemap Analysis
  console.log('\n🗺️ PHASE 2: SITEMAP.XML ANALYSIS');
  console.log('─'.repeat(40));
  
  const sitemapUrls = ['/sitemap.xml', '/sitemap_index.xml'];
  
  for (const sitemapUrl of sitemapUrls) {
    try {
      const sitemapResponse = await fetch(sitemapUrl);
      if (sitemapResponse.ok) {
        audit.sitemap.exists = true;
        console.log(`✅ Sitemap found: ${sitemapUrl}`);
        
        const sitemapXml = await sitemapResponse.text();
        const sitemap = parseSitemap(sitemapXml);
        
        // Check for issues
        if (sitemap.urls.length === 0) {
          audit.sitemap.issues.push('Empty sitemap');
          audit.score -= 10;
        }
        
        if (sitemap.urls.length > 50000) {
          audit.sitemap.issues.push('Too many URLs (>50k)');
          audit.score -= 5;
        }
        
        // Check for HTTPS
        const httpUrls = sitemap.urls.filter(url => url.loc.startsWith('http://'));
        if (httpUrls.length > 0) {
          audit.sitemap.issues.push('Contains HTTP URLs');
          audit.score -= 5;
        }
        
        break;
      }
    } catch (e) {
      // Continue
    }
  }
  
  if (!audit.sitemap.exists) {
    console.log('❌ No sitemap found');
    audit.recommendations.push('Create XML sitemap');
    audit.score -= 15;
  }
  
  // Phase 3: Crawlability Test
  console.log('\n🕷️ PHASE 3: CRAWLABILITY TEST');
  console.log('─'.repeat(40));
  
  const testUrls = ['/', '/about', '/contact', '/products', '/services'];
  
  for (const path of testUrls) {
    try {
      const response = await fetch(path, { method: 'HEAD' });
      if (response.ok) {
        audit.crawlability.accessible++;
      } else if (response.status === 403 || response.status === 401) {
        audit.crawlability.blocked++;
      }
    } catch (e) {
      // Skip
    }
  }
  
  console.log(`Accessible pages: ${audit.crawlability.accessible}`);
  console.log(`Blocked pages: ${audit.crawlability.blocked}`);
  
  // Phase 4: Meta Tags
  console.log('\n🏷️ PHASE 4: META TAGS CHECK');
  console.log('─'.repeat(40));
  
  const metaRobots = document.querySelector('meta[name="robots"]');
  const metaDescription = document.querySelector('meta[name="description"]');
  const title = document.querySelector('title');
  
  if (metaRobots?.content.includes('noindex')) {
    console.log('❌ Page has noindex meta tag!');
    audit.score -= 30;
  } else {
    console.log('✅ Page is indexable');
  }
  
  if (!metaDescription) {
    console.log('⚠️ No meta description');
    audit.recommendations.push('Add meta descriptions');
    audit.score -= 5;
  }
  
  if (!title || title.textContent.length < 10) {
    console.log('⚠️ Missing or short title tag');
    audit.recommendations.push('Optimize title tags');
    audit.score -= 5;
  }
  
  // Final Score
  audit.score = Math.max(0, audit.score);
  
  console.log('\n📊 AUDIT SUMMARY');
  console.log('═'.repeat(50));
  console.log(`SEO Configuration Score: ${audit.score}/100`);
  console.log(`robots.txt: ${audit.robots.exists ? '✅' : '❌'}`);
  console.log(`sitemap.xml: ${audit.sitemap.exists ? '✅' : '❌'}`);
  console.log(`Security issues: ${audit.security.exposed.length}`);
  
  // Risk Level
  let risk = 'LOW';
  if (audit.score < 50) risk = 'HIGH';
  else if (audit.score < 70) risk = 'MEDIUM';
  
  console.log(`Risk Level: ${risk}`);
  
  // Recommendations
  console.log('\n💡 TOP RECOMMENDATIONS:');
  
  if (!audit.robots.exists) {
    console.log('1. Create robots.txt to control crawling');
  }
  if (!audit.sitemap.exists) {
    console.log('2. Create XML sitemap for better discovery');
  }
  if (audit.security.exposed.length > 0) {
    console.log('3. Don\'t expose sensitive paths in robots.txt');
  }
  if (audit.robots.issues.length > 0) {
    console.log('4. Fix robots.txt issues');
  }
  if (audit.sitemap.issues.length > 0) {
    console.log('5. Fix sitemap issues');
  }
  
  audit.recommendations.forEach((rec, i) => {
    console.log(`${i + 6}. ${rec}`);
  });
  
  return audit;
}

// Run the complete audit
completeSEOAudit();

The Bottom Line

robots.txt and sitemap.xml are your SEO control panel:

• robots.txt advertises hidden content - Every blocked path is now public
• Sitemaps reveal site structure - Competitors can map your entire site
• Misconfigurations kill SEO - One wrong line blocks all traffic
• Security through obscurity fails - Don't hide sensitive URLs in robots.txt
• Regular audits are essential - Sites change, configs drift

Configure carefully. Monitor constantly. These files control your search visibility.

---

Analyze SEO configuration instantly: Fusebox checks robots.txt, analyzes sitemaps, detects crawl issues, and monitors SEO health while you browse. Optimize your search visibility. $29 one-time purchase.