AI Agent Implementation Guide

https://[domain]/llmtag.txt

import requests
import re
from typing import Dict, List, Optional

class LLMTAGParser:
    def __init__(self):
        self.cache = {}
    
    def get_policies(self, domain: str, user_agent: str, path: str) -> Dict:
        """Get LLMTAG policies for a domain, user agent, and path."""
        
        # Check cache first
        cache_key = f"{domain}:{user_agent}:{path}"
        if cache_key in self.cache:
            return self.cache[cache_key]
        
        # Fetch llmtag.txt
        try:
            response = requests.get(f"https://{domain}/llmtag.txt", timeout=10)
            if response.status_code != 200:
                return self._get_default_policies()
            
            content = response.text
            policies = self._parse_llmtag(content, user_agent, path)
            
            # Cache the result
            self.cache[cache_key] = policies
            return policies
            
        except Exception as e:
            print(f"Error fetching llmtag.txt for {domain}: {e}")
            return self._get_default_policies()
    
    def _parse_llmtag(self, content: str, user_agent: str, path: str) -> Dict:
        """Parse llmtag.txt content and return applicable policies."""
        
        lines = content.strip().split('\n')
        policies = self._get_default_policies()
        
        current_scope = None
        current_agent = None
        current_path = None
        
        for line in lines:
            line = line.strip()
            if not line or line.startswith('#'):
                continue
            
            if line.startswith('spec_version:'):
                # Validate specification version
                version = line.split(':', 1)[1].strip()
                if version != '3.0':
                    print(f"Unsupported LLMTAG version: {version}")
                    return self._get_default_policies()
            
            elif line.startswith('User-agent:'):
                current_agent = line.split(':', 1)[1].strip()
                current_scope = 'agent'
            
            elif line.startswith('Path:'):
                current_path = line.split(':', 1)[1].strip()
                current_scope = 'path'
            
            elif line.startswith('ai_training_data:'):
                value = line.split(':', 1)[1].strip()
                if self._applies_to_current_scope(user_agent, path, current_agent, current_path):
                    policies['ai_training_data'] = value
            
            elif line.startswith('ai_use:'):
                value = line.split(':', 1)[1].strip()
                if self._applies_to_current_scope(user_agent, path, current_agent, current_path):
                    policies['ai_use'] = [v.strip() for v in value.split(',')]
        
        return policies
    
    def _applies_to_current_scope(self, user_agent: str, path: str, 
                                current_agent: str, current_path: str) -> bool:
        """Check if current scope applies to the given user agent and path."""
        
        if current_agent and current_agent.lower() not in user_agent.lower():
            return False
        
        if current_path and not path.startswith(current_path):
            return False
        
        return True
    
    def _get_default_policies(self) -> Dict:
        """Return default policies when no llmtag.txt is found."""
        return {
            'ai_training_data': 'allow',
            'ai_use': ['search_indexing']
        }

# Usage example
parser = LLMTAGParser()
policies = parser.get_policies('example.com', 'MyAI-Bot/1.0', '/blog/post-1')

if policies['ai_training_data'] == 'disallow':
    print("AI training not allowed for this content")
else:
    print("AI training allowed")

if 'generative_synthesis' not in policies['ai_use']:
    print("Generative synthesis not allowed")
else:
    print("Generative synthesis allowed")

class LLMTAGParser {
    constructor() {
        this.cache = new Map();
    }
    
    async getPolicies(domain, userAgent, path) {
        // Check cache first
        const cacheKey = `${domain}:${userAgent}:${path}`;
        if (this.cache.has(cacheKey)) {
            return this.cache.get(cacheKey);
        }
        
        try {
            // Fetch llmtag.txt
            const response = await fetch(`https://${domain}/llmtag.txt`, {
                method: 'GET',
                timeout: 10000
            });
            
            if (!response.ok) {
                return this.getDefaultPolicies();
            }
            
            const content = await response.text();
            const policies = this.parseLLMTAG(content, userAgent, path);
            
            // Cache the result
            this.cache.set(cacheKey, policies);
            return policies;
            
        } catch (error) {
            console.error(`Error fetching llmtag.txt for ${domain}:`, error);
            return this.getDefaultPolicies();
        }
    }
    
    parseLLMTAG(content, userAgent, path) {
        const lines = content.trim().split('\n');
        const policies = this.getDefaultPolicies();
        
        let currentScope = null;
        let currentAgent = null;
        let currentPath = null;
        
        for (const line of lines) {
            const trimmedLine = line.trim();
            if (!trimmedLine || trimmedLine.startsWith('#')) {
                continue;
            }
            
            if (trimmedLine.startsWith('spec_version:')) {
                const version = trimmedLine.split(':')[1].trim();
                if (version !== '3.0') {
                    console.warn(`Unsupported LLMTAG version: ${version}`);
                    return this.getDefaultPolicies();
                }
            }
            
            else if (trimmedLine.startsWith('User-agent:')) {
                currentAgent = trimmedLine.split(':')[1].trim();
                currentScope = 'agent';
            }
            
            else if (trimmedLine.startsWith('Path:')) {
                currentPath = trimmedLine.split(':')[1].trim();
                currentScope = 'path';
            }
            
            else if (trimmedLine.startsWith('ai_training_data:')) {
                const value = trimmedLine.split(':')[1].trim();
                if (this.appliesToCurrentScope(userAgent, path, currentAgent, currentPath)) {
                    policies.ai_training_data = value;
                }
            }
            
            else if (trimmedLine.startsWith('ai_use:')) {
                const value = trimmedLine.split(':')[1].trim();
                if (this.appliesToCurrentScope(userAgent, path, currentAgent, currentPath)) {
                    policies.ai_use = value.split(',').map(v => v.trim());
                }
            }
        }
        
        return policies;
    }
    
    appliesToCurrentScope(userAgent, path, currentAgent, currentPath) {
        if (currentAgent && !userAgent.toLowerCase().includes(currentAgent.toLowerCase())) {
            return false;
        }
        
        if (currentPath && !path.startsWith(currentPath)) {
            return false;
        }
        
        return true;
    }
    
    getDefaultPolicies() {
        return {
            ai_training_data: 'allow',
            ai_use: ['search_indexing']
        };
    }
}

// Usage example
const parser = new LLMTAGParser();
const policies = await parser.getPolicies('example.com', 'MyAI-Bot/1.0', '/blog/post-1');

if (policies.ai_training_data === 'disallow') {
    console.log('AI training not allowed for this content');
} else {
    console.log('AI training allowed');
}

if (!policies.ai_use.includes('generative_synthesis')) {
    console.log('Generative synthesis not allowed');
} else {
    console.log('Generative synthesis allowed');
}