Add project files:

- Add database initialization scripts
- Add configuration files
- Add documentation
- Add public assets
- Add source code structure
- Update README

src/app.ts

@@ -1,38 +1,48 @@
-import { startWebSocketServer } from './websocket';
 import express from 'express';
 import cors from 'cors';
-import http from 'http';
-import routes from './routes';
-import { getPort } from './config';
-import logger from './utils/logger';
-
-const port = getPort();
+import path from 'path';
+import './config/env'; // Load environment variables first
+import apiRoutes from './routes/api';
+import { HealthCheckService } from './lib/services/healthCheck';
 
 const app = express();
-const server = http.createServer(app);
+const port = process.env.PORT || 3000;
 
-const corsOptions = {
-  origin: '*',
-};
-
-app.use(cors(corsOptions));
+// Middleware
+app.use(cors());
 app.use(express.json());
 
-app.use('/api', routes);
-app.get('/api', (_, res) => {
-  res.status(200).json({ status: 'ok' });
+// API routes first
+app.use('/api', apiRoutes);
+
+// Then static files
+app.use(express.static(path.join(__dirname, '../public')));
+
+// Finally, catch-all route for SPA
+app.get('*', (req, res) => {
+  res.sendFile(path.join(__dirname, '../public/index.html'));
 });
 
-server.listen(port, () => {
-  logger.info(`Server is running on port ${port}`);
-});
+// Start server with health checks
+async function startServer() {
+  console.log('\n🔍 Checking required services...');
+  
+  const ollamaStatus = await HealthCheckService.checkOllama();
+  const searxngStatus = await HealthCheckService.checkSearxNG();
+  const supabaseStatus = await HealthCheckService.checkSupabase();
 
-startWebSocketServer(server);
+  console.log('\n📊 Service Status:');
+  console.log('- Ollama:', ollamaStatus ? '✅ Running' : '❌ Not Running');
+  console.log('- SearxNG:', searxngStatus ? '✅ Running' : '❌ Not Running');
+  console.log('- Supabase:', supabaseStatus ? '✅ Connected' : '❌ Not Connected');
 
-process.on('uncaughtException', (err, origin) => {
-  logger.error(`Uncaught Exception at ${origin}: ${err}`);
-});
+  app.listen(port, () => {
+    console.log(`\n🚀 Server running at http://localhost:${port}`);
+    console.log('-------------------------------------------');
+  });
+}
 
-process.on('unhandledRejection', (reason, promise) => {
-  logger.error(`Unhandled Rejection at: ${promise}, reason: ${reason}`);
+startServer().catch(error => {
+  console.error('Failed to start server:', error);
+  process.exit(1);
 });

src/config.ts

@@ -77,3 +77,16 @@ export const updateConfig = (config: RecursivePartial<Config>) => {
     toml.stringify(config),
   );
 };
+
+export const config = {
+  ollama: {
+    url: process.env.OLLAMA_URL || 'http://localhost:11434',
+    model: process.env.OLLAMA_MODEL || 'mistral',
+    options: {
+      temperature: 0.1,
+      top_p: 0.9,
+      timeout: 30000 // 30 seconds timeout
+    }
+  },
+  // ... other config
+};

src/config/env.ts

@@ -0,0 +1,68 @@
+import { config } from 'dotenv';
+import { z } from 'zod';
+
+config();
+
+// Define the environment schema
+const envSchema = z.object({
+  PORT: z.string().default('3000'),
+  NODE_ENV: z.string().default('development'),
+  SUPABASE_URL: z.string(),
+  SUPABASE_KEY: z.string(),
+  OLLAMA_URL: z.string().default('http://localhost:11434'),
+  OLLAMA_MODEL: z.string().default('llama2'),
+  SEARXNG_URL: z.string().default('http://localhost:4000'),
+  SEARXNG_INSTANCES: z.string().default('["http://localhost:4000"]'),
+  MAX_RESULTS_PER_QUERY: z.string().default('50'),
+  CACHE_DURATION_HOURS: z.string().default('24'),
+  CACHE_DURATION_DAYS: z.string().default('7')
+});
+
+// Define the final environment type
+export interface EnvConfig {
+  PORT: string;
+  NODE_ENV: string;
+  searxng: {
+    currentUrl: string;
+    instances: string[];
+  };
+  ollama: {
+    url: string;
+    model: string;
+  };
+  supabase: {
+    url: string;
+    anonKey: string;
+  };
+  cache: {
+    maxResultsPerQuery: number;
+    durationHours: number;
+    durationDays: number;
+  };
+}
+
+// Parse and transform the environment variables
+const rawEnv = envSchema.parse(process.env);
+
+// Create the final environment object with parsed configurations
+export const env: EnvConfig = {
+  PORT: rawEnv.PORT,
+  NODE_ENV: rawEnv.NODE_ENV,
+  searxng: {
+    currentUrl: rawEnv.SEARXNG_URL,
+    instances: JSON.parse(rawEnv.SEARXNG_INSTANCES)
+  },
+  ollama: {
+    url: rawEnv.OLLAMA_URL,
+    model: rawEnv.OLLAMA_MODEL
+  },
+  supabase: {
+    url: rawEnv.SUPABASE_URL,
+    anonKey: rawEnv.SUPABASE_KEY
+  },
+  cache: {
+    maxResultsPerQuery: parseInt(rawEnv.MAX_RESULTS_PER_QUERY),
+    durationHours: parseInt(rawEnv.CACHE_DURATION_HOURS),
+    durationDays: parseInt(rawEnv.CACHE_DURATION_DAYS)
+  }
+}; 

src/config/index.ts

@@ -0,0 +1,77 @@
+import dotenv from 'dotenv';
+import path from 'path';
+
+// Load .env file
+dotenv.config({ path: path.resolve(__dirname, '../../.env') });
+
+export interface Config {
+  supabase: {
+    url: string;
+    anonKey: string;
+  };
+  server: {
+    port: number;
+    nodeEnv: string;
+  };
+  search: {
+    maxResultsPerQuery: number;
+    cacheDurationHours: number;
+    searxngUrl?: string;
+  };
+  rateLimit: {
+    windowMs: number;
+    maxRequests: number;
+  };
+  security: {
+    corsOrigin: string;
+    jwtSecret: string;
+  };
+  proxy?: {
+    http?: string;
+    https?: string;
+  };
+  logging: {
+    level: string;
+  };
+}
+
+const config: Config = {
+  supabase: {
+    url: process.env.SUPABASE_URL || '',
+    anonKey: process.env.SUPABASE_ANON_KEY || '',
+  },
+  server: {
+    port: parseInt(process.env.PORT || '3000', 10),
+    nodeEnv: process.env.NODE_ENV || 'development',
+  },
+  search: {
+    maxResultsPerQuery: parseInt(process.env.MAX_RESULTS_PER_QUERY || '20', 10),
+    cacheDurationHours: parseInt(process.env.CACHE_DURATION_HOURS || '24', 10),
+    searxngUrl: process.env.SEARXNG_URL
+  },
+  rateLimit: {
+    windowMs: parseInt(process.env.RATE_LIMIT_WINDOW_MS || '900000', 10),
+    maxRequests: parseInt(process.env.RATE_LIMIT_MAX_REQUESTS || '100', 10),
+  },
+  security: {
+    corsOrigin: process.env.CORS_ORIGIN || 'http://localhost:3000',
+    jwtSecret: process.env.JWT_SECRET || 'your_jwt_secret_key',
+  },
+  logging: {
+    level: process.env.LOG_LEVEL || 'info',
+  },
+};
+
+// Validate required configuration
+const validateConfig = () => {
+  if (!config.supabase.url) {
+    throw new Error('SUPABASE_URL is required');
+  }
+  if (!config.supabase.anonKey) {
+    throw new Error('SUPABASE_ANON_KEY is required');
+  }
+};
+
+validateConfig();
+
+export { config }; 

src/lib/categories.ts

@@ -0,0 +1,116 @@
+export interface Category {
+  id: string;
+  name: string;
+  icon: string;
+  subcategories: SubCategory[];
+}
+
+export interface SubCategory {
+  id: string;
+  name: string;
+}
+
+export const categories: Category[] = [
+  {
+    id: 'real-estate-pros',
+    name: 'Real Estate Professionals',
+    icon: '🏢',
+    subcategories: [
+      { id: 'wholesalers', name: 'Real Estate Wholesalers' },
+      { id: 'agents', name: 'Real Estate Agents' },
+      { id: 'attorneys', name: 'Real Estate Attorneys' },
+      { id: 'scouts', name: 'Property Scouts' },
+      { id: 'brokers', name: 'Real Estate Brokers' },
+      { id: 'consultants', name: 'Real Estate Consultants' }
+    ]
+  },
+  {
+    id: 'legal-title',
+    name: 'Legal & Title Services',
+    icon: '⚖️',
+    subcategories: [
+      { id: 'title-companies', name: 'Title Companies' },
+      { id: 'closing-attorneys', name: 'Closing Attorneys' },
+      { id: 'zoning-consultants', name: 'Zoning Consultants' },
+      { id: 'probate-specialists', name: 'Probate Specialists' },
+      { id: 'eviction-specialists', name: 'Eviction Specialists' }
+    ]
+  },
+  {
+    id: 'financial',
+    name: 'Financial Services',
+    icon: '💰',
+    subcategories: [
+      { id: 'hard-money', name: 'Hard Money Lenders' },
+      { id: 'private-equity', name: 'Private Equity Investors' },
+      { id: 'mortgage-brokers', name: 'Mortgage Brokers' },
+      { id: 'tax-advisors', name: 'Tax Advisors' },
+      { id: 'appraisers', name: 'Appraisers' }
+    ]
+  },
+  {
+    id: 'contractors',
+    name: 'Specialist Contractors',
+    icon: '🔨',
+    subcategories: [
+      { id: 'general', name: 'General Contractors' },
+      { id: 'plumbers', name: 'Plumbers' },
+      { id: 'electricians', name: 'Electricians' },
+      { id: 'hvac', name: 'HVAC Technicians' },
+      { id: 'roofers', name: 'Roofers' },
+      { id: 'foundation', name: 'Foundation Specialists' },
+      { id: 'asbestos', name: 'Asbestos Removal' },
+      { id: 'mold', name: 'Mold Remediation' }
+    ]
+  },
+  {
+    id: 'property-services',
+    name: 'Property Services',
+    icon: '🏠',
+    subcategories: [
+      { id: 'surveyors', name: 'Surveyors' },
+      { id: 'inspectors', name: 'Inspectors' },
+      { id: 'property-managers', name: 'Property Managers' },
+      { id: 'environmental', name: 'Environmental Consultants' },
+      { id: 'junk-removal', name: 'Junk Removal Services' },
+      { id: 'cleaning', name: 'Property Cleaning' }
+    ]
+  },
+  {
+    id: 'marketing',
+    name: 'Marketing & Lead Gen',
+    icon: '📢',
+    subcategories: [
+      { id: 'direct-mail', name: 'Direct Mail Services' },
+      { id: 'social-media', name: 'Social Media Marketing' },
+      { id: 'seo', name: 'SEO Specialists' },
+      { id: 'ppc', name: 'PPC Advertising' },
+      { id: 'lead-gen', name: 'Lead Generation' },
+      { id: 'skip-tracing', name: 'Skip Tracing Services' }
+    ]
+  },
+  {
+    id: 'data-tech',
+    name: 'Data & Technology',
+    icon: '💻',
+    subcategories: [
+      { id: 'data-providers', name: 'Property Data Providers' },
+      { id: 'crm', name: 'CRM Systems' },
+      { id: 'valuation', name: 'Valuation Tools' },
+      { id: 'virtual-tours', name: 'Virtual Tour Services' },
+      { id: 'automation', name: 'Automation Tools' }
+    ]
+  },
+  {
+    id: 'specialty',
+    name: 'Specialty Services',
+    icon: '🎯',
+    subcategories: [
+      { id: 'auction', name: 'Auction Companies' },
+      { id: 'relocation', name: 'Relocation Services' },
+      { id: 'staging', name: 'Home Staging' },
+      { id: 'photography', name: 'Real Estate Photography' },
+      { id: 'virtual-assistant', name: 'Virtual Assistants' }
+    ]
+  }
+]; 

src/lib/db/optOutDb.ts

@@ -0,0 +1,51 @@
+import { Database } from 'better-sqlite3';
+import path from 'path';
+
+interface OptOutEntry {
+  domain: string;
+  email: string;
+  reason?: string;
+  timestamp: Date;
+}
+
+export class OptOutDatabase {
+  private db: Database;
+
+  constructor() {
+    this.db = new Database(path.join(__dirname, '../../../data/optout.db'));
+    this.initializeDatabase();
+  }
+
+  private initializeDatabase() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS opt_outs (
+        domain TEXT PRIMARY KEY,
+        email TEXT NOT NULL,
+        reason TEXT,
+        timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+      CREATE INDEX IF NOT EXISTS idx_domain ON opt_outs(domain);
+    `);
+  }
+
+  async addOptOut(entry: OptOutEntry): Promise<void> {
+    const stmt = this.db.prepare(
+      'INSERT OR REPLACE INTO opt_outs (domain, email, reason, timestamp) VALUES (?, ?, ?, ?)'
+    );
+    stmt.run(entry.domain, entry.email, entry.reason, entry.timestamp.toISOString());
+  }
+
+  isOptedOut(domain: string): boolean {
+    const stmt = this.db.prepare('SELECT 1 FROM opt_outs WHERE domain = ?');
+    return stmt.get(domain) !== undefined;
+  }
+
+  removeOptOut(domain: string): void {
+    const stmt = this.db.prepare('DELETE FROM opt_outs WHERE domain = ?');
+    stmt.run(domain);
+  }
+
+  getOptOutList(): OptOutEntry[] {
+    return this.db.prepare('SELECT * FROM opt_outs').all();
+  }
+} 

src/lib/db/supabase.ts

@@ -0,0 +1,74 @@
+import { createClient } from '@supabase/supabase-js';
+import { BusinessData } from '../searxng';
+import { env } from '../../config/env';
+
+// Create the Supabase client with validated environment variables
+export const supabase = createClient(
+  env.supabase.url,
+  env.supabase.anonKey,
+  {
+    auth: {
+      persistSession: false // Since this is a server environment
+    }
+  }
+);
+
+// Define the cache record type
+export interface CacheRecord {
+  id: string;
+  query: string;
+  results: BusinessData[];
+  location: string;
+  category: string;
+  created_at: string;
+  updated_at: string;
+  expires_at: string;
+}
+
+// Export database helper functions
+export async function getCacheEntry(
+  category: string,
+  location: string
+): Promise<CacheRecord | null> {
+  const { data, error } = await supabase
+    .from('search_cache')
+    .select('*')
+    .eq('category', category.toLowerCase())
+    .eq('location', location.toLowerCase())
+    .gt('expires_at', new Date().toISOString())
+    .order('created_at', { ascending: false })
+    .limit(1)
+    .single();
+
+  if (error) {
+    console.error('Cache lookup failed:', error);
+    return null;
+  }
+
+  return data;
+}
+
+export async function saveCacheEntry(
+  category: string,
+  location: string,
+  results: BusinessData[],
+  expiresInDays: number = 7
+): Promise<void> {
+  const expiresAt = new Date();
+  expiresAt.setDate(expiresAt.getDate() + expiresInDays);
+
+  const { error } = await supabase
+    .from('search_cache')
+    .insert({
+      query: `${category} in ${location}`,
+      category: category.toLowerCase(),
+      location: location.toLowerCase(),
+      results,
+      expires_at: expiresAt.toISOString()
+    });
+
+  if (error) {
+    console.error('Failed to save cache entry:', error);
+    throw error;
+  }
+} 

src/lib/emailScraper.ts

@@ -0,0 +1,195 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { Cache } from './utils/cache';
+import { RateLimiter } from './utils/rateLimiter';
+import robotsParser from 'robots-parser';
+
+interface ScrapingResult {
+  emails: string[];
+  phones: string[];
+  addresses: string[];
+  socialLinks: string[];
+  source: string;
+  timestamp: Date;
+  attribution: string;
+}
+
+export class EmailScraper {
+  private cache: Cache<ScrapingResult>;
+  private rateLimiter: RateLimiter;
+  private robotsCache = new Map<string, any>();
+
+  constructor(private options = { 
+    timeout: 5000,
+    cacheTTL: 60,
+    rateLimit: { windowMs: 60000, maxRequests: 10 }, // More conservative rate limiting
+    userAgent: 'BizSearch/1.0 (+https://your-domain.com/about) - Business Directory Service'
+  }) {
+    this.cache = new Cache<ScrapingResult>(options.cacheTTL);
+    this.rateLimiter = new RateLimiter(options.rateLimit.windowMs, options.rateLimit.maxRequests);
+  }
+
+  private async checkRobotsPermission(url: string): Promise<boolean> {
+    try {
+      const { protocol, host } = new URL(url);
+      const robotsUrl = `${protocol}//${host}/robots.txt`;
+      
+      let parser = this.robotsCache.get(host);
+      if (!parser) {
+        const response = await axios.get(robotsUrl);
+        parser = robotsParser(robotsUrl, response.data);
+        this.robotsCache.set(host, parser);
+      }
+
+      return parser.isAllowed(url, this.options.userAgent);
+    } catch (error) {
+      console.warn(`Could not check robots.txt for ${url}:`, error);
+      return true; // Assume allowed if robots.txt is unavailable
+    }
+  }
+
+  async scrapeEmails(url: string): Promise<ScrapingResult> {
+    // Check cache first
+    const cached = this.cache.get(url);
+    if (cached) return cached;
+
+    // Check robots.txt
+    const allowed = await this.checkRobotsPermission(url);
+    if (!allowed) {
+      console.log(`Respecting robots.txt disallow for ${url}`);
+      return {
+        emails: [],
+        phones: [],
+        addresses: [],
+        socialLinks: [],
+        source: url,
+        timestamp: new Date(),
+        attribution: 'Restricted by robots.txt'
+      };
+    }
+
+    // Wait for rate limiting slot
+    await this.rateLimiter.waitForSlot();
+
+    try {
+      const response = await axios.get(url, {
+        timeout: this.options.timeout,
+        headers: {
+          'User-Agent': this.options.userAgent,
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        }
+      });
+
+      // Check for noindex meta tag
+      const $ = cheerio.load(response.data);
+      if ($('meta[name="robots"][content*="noindex"]').length > 0) {
+        return {
+          emails: [],
+          phones: [],
+          addresses: [],
+          socialLinks: [],
+          source: url,
+          timestamp: new Date(),
+          attribution: 'Respecting noindex directive'
+        };
+      }
+
+      // Only extract contact information from public contact pages or structured data
+      const isContactPage = /contact|about/i.test(url) || 
+                          $('h1, h2').text().toLowerCase().includes('contact');
+
+      const result = {
+        emails: new Set<string>(),
+        phones: new Set<string>(),
+        addresses: new Set<string>(),
+        socialLinks: new Set<string>(),
+        source: url,
+        timestamp: new Date(),
+        attribution: `Data from public business listing at ${new URL(url).hostname}`
+      };
+
+      // Extract from structured data (Schema.org)
+      $('script[type="application/ld+json"]').each((_, element) => {
+        try {
+          const data = JSON.parse($(element).html() || '{}');
+          if (data['@type'] === 'LocalBusiness' || data['@type'] === 'Organization') {
+            if (data.email) result.emails.add(data.email.toLowerCase());
+            if (data.telephone) result.phones.add(this.formatPhoneNumber(data.telephone));
+            if (data.address) {
+              const fullAddress = this.formatAddress(data.address);
+              if (fullAddress) result.addresses.add(fullAddress);
+            }
+          }
+        } catch (e) {
+          console.error('Error parsing JSON-LD:', e);
+        }
+      });
+
+      // Only scrape additional info if it's a contact page
+      if (isContactPage) {
+        // Extract clearly marked contact information
+        $('[itemprop="email"], .contact-email, .email').each((_, element) => {
+          const email = $(element).text().trim();
+          if (this.isValidEmail(email)) {
+            result.emails.add(email.toLowerCase());
+          }
+        });
+
+        $('[itemprop="telephone"], .phone, .contact-phone').each((_, element) => {
+          const phone = $(element).text().trim();
+          const formatted = this.formatPhoneNumber(phone);
+          if (formatted) result.phones.add(formatted);
+        });
+      }
+
+      const finalResult = {
+        ...result,
+        emails: Array.from(result.emails),
+        phones: Array.from(result.phones),
+        addresses: Array.from(result.addresses),
+        socialLinks: Array.from(result.socialLinks)
+      };
+
+      this.cache.set(url, finalResult);
+      return finalResult;
+
+    } catch (error) {
+      console.error(`Failed to scrape ${url}:`, error);
+      return {
+        emails: [],
+        phones: [],
+        addresses: [],
+        socialLinks: [],
+        source: url,
+        timestamp: new Date(),
+        attribution: 'Error accessing page'
+      };
+    }
+  }
+
+  private isValidEmail(email: string): boolean {
+    return /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.test(email);
+  }
+
+  private formatPhoneNumber(phone: string): string {
+    const digits = phone.replace(/\D/g, '');
+    if (digits.length === 10) {
+      return `(${digits.slice(0,3)}) ${digits.slice(3,6)}-${digits.slice(6)}`;
+    }
+    return phone;
+  }
+
+  private formatAddress(address: any): string | null {
+    if (typeof address === 'string') return address;
+    if (typeof address === 'object') {
+      const parts = [
+        address.streetAddress,
+        address.addressLocality,
+        address.addressRegion,
+        address.postalCode
+      ].filter(Boolean);
+      if (parts.length > 0) return parts.join(', ');
+    }
+    return null;
+  }
+} 

src/lib/providers/business/index.ts

@@ -0,0 +1,19 @@
+import { Business, SearchParams } from '../../../types/business';
+import { WebScraperProvider } from './webScraper';
+
+export class BusinessProvider {
+  private scraper: WebScraperProvider;
+
+  constructor() {
+    this.scraper = new WebScraperProvider();
+  }
+
+  async search(params: SearchParams): Promise<Business[]> {
+    return this.scraper.search(params);
+  }
+
+  async getDetails(businessId: string): Promise<Business | null> {
+    // Implement detailed business lookup using stored data or additional scraping
+    return null;
+  }
+} 

src/lib/providers/business/webScraper.ts

@@ -0,0 +1,111 @@
+import { Business, SearchParams } from '../../../types/business';
+import { searchWeb } from '../search'; // This is Perplexica's existing search function
+import { parseHTML } from '../utils/parser';
+
+export class WebScraperProvider {
+  async search(params: SearchParams): Promise<Business[]> {
+    const searchQueries = this.generateQueries(params);
+    const businesses: Business[] = [];
+
+    for (const query of searchQueries) {
+      // Use Perplexica's existing search functionality
+      const results = await searchWeb(query, {
+        maxResults: 20,
+        type: 'general'  // or 'news' depending on what we want
+      });
+
+      for (const result of results) {
+        try {
+          const html = await fetch(result.url).then(res => res.text());
+          const businessData = await this.extractBusinessData(html, result.url);
+          if (businessData) {
+            businesses.push(businessData);
+          }
+        } catch (error) {
+          console.error(`Failed to extract data from ${result.url}:`, error);
+        }
+      }
+    }
+
+    return this.deduplicateBusinesses(businesses);
+  }
+
+  private generateQueries(params: SearchParams): string[] {
+    const { location, category } = params;
+    return [
+      `${category} in ${location}`,
+      `${category} business ${location}`,
+      `best ${category} near ${location}`,
+      `${category} services ${location} reviews`
+    ];
+  }
+
+  private async extractBusinessData(html: string, sourceUrl: string): Promise<Business | null> {
+    const $ = parseHTML(html);
+    
+    // Different extraction logic based on source
+    if (sourceUrl.includes('yelp.com')) {
+      return this.extractYelpData($);
+    } else if (sourceUrl.includes('yellowpages.com')) {
+      return this.extractYellowPagesData($);
+    }
+    // ... other source-specific extractors
+
+    return null;
+  }
+
+  private extractYelpData($: any): Business | null {
+    try {
+      return {
+        id: crypto.randomUUID(),
+        name: $('.business-name').text().trim(),
+        phone: $('.phone-number').text().trim(),
+        address: $('.address').text().trim(),
+        city: $('.city').text().trim(),
+        state: $('.state').text().trim(),
+        zip: $('.zip').text().trim(),
+        category: $('.category-str-list').text().split(',').map(s => s.trim()),
+        rating: parseFloat($('.rating').text()),
+        reviewCount: parseInt($('.review-count').text()),
+        services: $('.services-list').text().split(',').map(s => s.trim()),
+        hours: this.extractHours($),
+        website: $('.website-link').attr('href'),
+        verified: false,
+        lastUpdated: new Date()
+      };
+    } catch (error) {
+      return null;
+    }
+  }
+
+  private deduplicateBusinesses(businesses: Business[]): Business[] {
+    // Group by phone number and address to identify duplicates
+    const uniqueBusinesses = new Map<string, Business>();
+    
+    for (const business of businesses) {
+      const key = `${business.phone}-${business.address}`.toLowerCase();
+      if (!uniqueBusinesses.has(key)) {
+        uniqueBusinesses.set(key, business);
+      } else {
+        // Merge data if we have additional information
+        const existing = uniqueBusinesses.get(key)!;
+        uniqueBusinesses.set(key, this.mergeBusinessData(existing, business));
+      }
+    }
+
+    return Array.from(uniqueBusinesses.values());
+  }
+
+  private mergeBusinessData(existing: Business, newData: Business): Business {
+    return {
+      ...existing,
+      services: [...new Set([...existing.services, ...newData.services])],
+      rating: (existing.rating + newData.rating) / 2,
+      reviewCount: existing.reviewCount + newData.reviewCount,
+      // Keep the most complete data for other fields
+      website: existing.website || newData.website,
+      email: existing.email || newData.email,
+      hours: existing.hours || newData.hours
+    };
+  }
+} 

src/lib/search.ts

@@ -0,0 +1,54 @@
+import axios from 'axios';
+import { config } from '../config';
+
+interface SearchOptions {
+  maxResults?: number;
+  type?: 'general' | 'news';
+  engines?: string[];
+}
+
+interface SearchResult {
+  url: string;
+  title: string;
+  content: string;
+  score?: number;
+}
+
+export async function searchWeb(
+  query: string,
+  options: SearchOptions = {}
+): Promise<SearchResult[]> {
+  const {
+    maxResults = 20,
+    type = 'general',
+    engines = ['google', 'bing', 'duckduckgo']
+  } = options;
+
+  try {
+    const response = await axios.get(`${config.search.searxngUrl || process.env.SEARXNG_URL}/search`, {
+      params: {
+        q: query,
+        format: 'json',
+        categories: type,
+        engines: engines.join(','),
+        limit: maxResults
+      }
+    });
+
+    if (!response.data || !response.data.results) {
+      console.error('Invalid response from SearxNG:', response.data);
+      return [];
+    }
+
+    return response.data.results.map((result: any) => ({
+      url: result.url,
+      title: result.title,
+      content: result.content || result.snippet || '',
+      score: result.score
+    }));
+
+  } catch (error) {
+    console.error('Search failed:', error);
+    throw error;
+  }
+} 

src/lib/services/businessCrawler.ts

@@ -0,0 +1,111 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { Cache } from '../utils/cache';
+import { RateLimiter } from '../utils/rateLimiter';
+
+interface CrawlResult {
+  mainContent: string;
+  contactInfo: string;
+  aboutInfo: string;
+  structuredData: any;
+}
+
+export class BusinessCrawler {
+  private cache: Cache<CrawlResult>;
+  private rateLimiter: RateLimiter;
+
+  constructor() {
+    this.cache = new Cache<CrawlResult>(60); // 1 hour cache
+    this.rateLimiter = new RateLimiter();
+  }
+
+  async crawlBusinessSite(url: string): Promise<CrawlResult> {
+    // Check cache first
+    const cached = this.cache.get(url);
+    if (cached) return cached;
+
+    await this.rateLimiter.waitForSlot();
+
+    try {
+      const mainPage = await this.fetchPage(url);
+      const $ = cheerio.load(mainPage);
+      
+      // Get all important URLs
+      const contactUrl = this.findContactPage($, url);
+      const aboutUrl = this.findAboutPage($, url);
+
+      // Crawl additional pages
+      const [contactPage, aboutPage] = await Promise.all([
+        contactUrl ? this.fetchPage(contactUrl) : '',
+        aboutUrl ? this.fetchPage(aboutUrl) : ''
+      ]);
+
+      // Extract structured data
+      const structuredData = this.extractStructuredData($);
+
+      const result = {
+        mainContent: $('body').text(),
+        contactInfo: contactPage,
+        aboutInfo: aboutPage,
+        structuredData
+      };
+
+      this.cache.set(url, result);
+      return result;
+    } catch (error) {
+      console.error(`Failed to crawl ${url}:`, error);
+      return {
+        mainContent: '',
+        contactInfo: '',
+        aboutInfo: '',
+        structuredData: {}
+      };
+    }
+  }
+
+  private async fetchPage(url: string): Promise<string> {
+    try {
+      const response = await axios.get(url, {
+        timeout: 10000,
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; BizSearch/1.0; +http://localhost:3000/about)',
+        }
+      });
+      return response.data;
+    } catch (error) {
+      console.error(`Failed to fetch ${url}:`, error);
+      return '';
+    }
+  }
+
+  private findContactPage($: cheerio.CheerioAPI, baseUrl: string): string | null {
+    const contactLinks = $('a[href*="contact"], a:contains("Contact")');
+    if (contactLinks.length > 0) {
+      const href = contactLinks.first().attr('href');
+      return href ? new URL(href, baseUrl).toString() : null;
+    }
+    return null;
+  }
+
+  private findAboutPage($: cheerio.CheerioAPI, baseUrl: string): string | null {
+    const aboutLinks = $('a[href*="about"], a:contains("About")');
+    if (aboutLinks.length > 0) {
+      const href = aboutLinks.first().attr('href');
+      return href ? new URL(href, baseUrl).toString() : null;
+    }
+    return null;
+  }
+
+  private extractStructuredData($: cheerio.CheerioAPI): any {
+    const structuredData: any[] = [];
+    $('script[type="application/ld+json"]').each((_, element) => {
+      try {
+        const data = JSON.parse($(element).html() || '{}');
+        structuredData.push(data);
+      } catch (error) {
+        console.error('Failed to parse structured data:', error);
+      }
+    });
+    return structuredData;
+  }
+} 

src/lib/services/cacheService.ts

@@ -0,0 +1,71 @@
+import { supabase } from '../supabase';
+import { BusinessData } from '../searxng';
+
+export class CacheService {
+  static async getCachedResults(category: string, location: string): Promise<BusinessData[] | null> {
+    try {
+      const { data, error } = await supabase
+        .from('search_cache')
+        .select('results')
+        .eq('category', category.toLowerCase())
+        .eq('location', location.toLowerCase())
+        .gt('expires_at', new Date().toISOString())
+        .order('created_at', { ascending: false })
+        .limit(1)
+        .single();
+
+      if (error) throw error;
+      return data ? data.results : null;
+    } catch (error) {
+      console.error('Cache lookup failed:', error);
+      return null;
+    }
+  }
+
+  static async cacheResults(
+    category: string, 
+    location: string, 
+    results: BusinessData[],
+    expiresInDays: number = 7
+  ): Promise<void> {
+    try {
+      const expiresAt = new Date();
+      expiresAt.setDate(expiresAt.getDate() + expiresInDays);
+
+      const { error } = await supabase
+        .from('search_cache')
+        .insert({
+          query: `${category} in ${location}`,
+          category: category.toLowerCase(),
+          location: location.toLowerCase(),
+          results,
+          expires_at: expiresAt.toISOString()
+        });
+
+      if (error) throw error;
+    } catch (error) {
+      console.error('Failed to cache results:', error);
+    }
+  }
+
+  static async updateCache(
+    category: string,
+    location: string,
+    newResults: BusinessData[]
+  ): Promise<void> {
+    try {
+      const { error } = await supabase
+        .from('search_cache')
+        .update({
+          results: newResults,
+          updated_at: new Date().toISOString()
+        })
+        .eq('category', category.toLowerCase())
+        .eq('location', location.toLowerCase());
+
+      if (error) throw error;
+    } catch (error) {
+      console.error('Failed to update cache:', error);
+    }
+  }
+} 

src/lib/services/dataValidation.ts

@@ -0,0 +1,107 @@
+import { OllamaService } from './ollamaService';
+
+interface ValidatedBusinessData {
+  name: string;
+  phone: string;
+  email: string;
+  address: string;
+  description: string;
+  hours?: string;
+  isValid: boolean;
+}
+
+export class DataValidationService {
+  private ollama: OllamaService;
+
+  constructor() {
+    this.ollama = new OllamaService();
+  }
+
+  async validateAndCleanData(rawText: string): Promise<ValidatedBusinessData> {
+    try {
+      const prompt = `
+        You are a business data validation expert. Extract and validate business information from the following text.
+        Return ONLY a JSON object with the following format, nothing else:
+        {
+          "name": "verified business name",
+          "phone": "formatted phone number or N/A",
+          "email": "verified email address or N/A",
+          "address": "verified physical address or N/A",
+          "description": "short business description",
+          "hours": "business hours if available",
+          "isValid": boolean
+        }
+
+        Rules:
+        1. Phone numbers should be in (XXX) XXX-XXXX format
+        2. Addresses should be properly formatted with street, city, state, zip
+        3. Remove any irrelevant text from descriptions
+        4. Set isValid to true only if name and at least one contact method is found
+        5. Clean up any obvious formatting issues
+        6. Validate email addresses for proper format
+
+        Text to analyze:
+        ${rawText}
+      `;
+
+      const response = await this.ollama.generateResponse(prompt);
+      
+      try {
+        // Find the JSON object in the response
+        const jsonMatch = response.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+          throw new Error('No JSON found in response');
+        }
+        
+        const result = JSON.parse(jsonMatch[0]);
+        return this.validateResult(result);
+      } catch (parseError) {
+        console.error('Failed to parse Ollama response:', parseError);
+        throw parseError;
+      }
+    } catch (error) {
+      console.error('Data validation failed:', error);
+      return {
+        name: 'Unknown',
+        phone: 'N/A',
+        email: 'N/A',
+        address: 'N/A',
+        description: '',
+        hours: '',
+        isValid: false
+      };
+    }
+  }
+
+  private validateResult(result: any): ValidatedBusinessData {
+    // Ensure all required fields are present
+    const validated: ValidatedBusinessData = {
+      name: this.cleanField(result.name) || 'Unknown',
+      phone: this.formatPhone(result.phone) || 'N/A',
+      email: this.cleanField(result.email) || 'N/A',
+      address: this.cleanField(result.address) || 'N/A',
+      description: this.cleanField(result.description) || '',
+      hours: this.cleanField(result.hours),
+      isValid: Boolean(result.isValid)
+    };
+
+    return validated;
+  }
+
+  private cleanField(value: any): string {
+    if (!value || typeof value !== 'string') return '';
+    return value.trim().replace(/\s+/g, ' ');
+  }
+
+  private formatPhone(phone: string): string {
+    if (!phone || phone === 'N/A') return 'N/A';
+    
+    // Extract digits
+    const digits = phone.replace(/\D/g, '');
+    if (digits.length === 10) {
+      return `(${digits.slice(0,3)}) ${digits.slice(3,6)}-${digits.slice(6)}`;
+    }
+    
+    return phone;
+  }
+} 

src/lib/services/healthCheck.ts

@@ -0,0 +1,53 @@
+import axios from 'axios';
+import { env } from '../../config/env';
+import { supabase } from '../supabase';
+
+export class HealthCheckService {
+  static async checkOllama(): Promise<boolean> {
+    try {
+      const response = await axios.get(`${env.ollama.url}/api/tags`);
+      return response.status === 200;
+    } catch (error) {
+      console.error('Ollama health check failed:', error);
+      return false;
+    }
+  }
+
+  static async checkSearxNG(): Promise<boolean> {
+    try {
+      const response = await axios.get(`${env.searxng.currentUrl}/config`);
+      return response.status === 200;
+    } catch (error) {
+      try {
+        const response = await axios.get(`${env.searxng.instances[0]}/config`);
+        return response.status === 200;
+      } catch (fallbackError) {
+        console.error('SearxNG health check failed:', error);
+        return false;
+      }
+    }
+  }
+
+  static async checkSupabase(): Promise<boolean> {
+    try {
+      console.log('Checking Supabase connection...');
+      console.log('URL:', env.supabase.url);
+
+      // Just check if we can connect and query, don't care about results
+      const { error } = await supabase
+        .from('businesses')
+        .select('count', { count: 'planned', head: true });
+
+      if (error) {
+        console.error('Supabase query error:', error);
+        return false;
+      }
+
+      console.log('Supabase connection successful');
+      return true;
+    } catch (error) {
+      console.error('Supabase connection failed:', error);
+      return false;
+    }
+  }
+} 

src/lib/services/ollamaService.ts

@@ -0,0 +1,36 @@
+import axios from 'axios';
+import { env } from '../../config/env';
+
+interface OllamaResponse {
+  response: string;
+  context?: number[];
+}
+
+export class OllamaService {
+  private url: string;
+  private model: string;
+
+  constructor() {
+    this.url = env.ollama.url;
+    this.model = env.ollama.model;
+  }
+
+  async complete(prompt: string): Promise<string> {
+    try {
+      const response = await axios.post(`${this.url}/api/generate`, {
+        model: this.model,
+        prompt: prompt,
+        stream: false,
+        options: {
+          temperature: 0.7,
+          top_p: 0.9
+        }
+      });
+
+      return response.data.response;
+    } catch (error) {
+      console.error('Ollama completion failed:', error);
+      throw error;
+    }
+  }
+} 

src/lib/services/supabaseService.ts

@@ -0,0 +1,93 @@
+import { createClient } from '@supabase/supabase-js';
+import { env } from '../../config/env';
+import { BusinessData } from '../searxng';
+
+export class SupabaseService {
+  private supabase;
+
+  constructor() {
+    this.supabase = createClient(env.supabase.url, env.supabase.anonKey);
+  }
+
+  async upsertBusinesses(businesses: BusinessData[]): Promise<void> {
+    try {
+      console.log('Upserting businesses to Supabase:', businesses.length);
+
+      for (const business of businesses) {
+        try {
+          // Create a unique identifier based on multiple properties
+          const identifier = [
+            business.name.toLowerCase(),
+            business.phone?.replace(/\D/g, ''),
+            business.address?.toLowerCase(),
+            business.website?.toLowerCase()
+          ]
+            .filter(Boolean)  // Remove empty values
+            .join('_')       // Join with underscore
+            .replace(/[^a-z0-9]/g, '_');  // Replace non-alphanumeric chars
+
+          // Log the data being inserted
+          console.log('Upserting business:', {
+            id: identifier,
+            name: business.name,
+            phone: business.phone,
+            email: business.email,
+            address: business.address,
+            rating: business.rating,
+            website: business.website,
+            location: business.location
+          });
+
+          // Check if business exists
+          const { data: existing, error: selectError } = await this.supabase
+            .from('businesses')
+            .select('rating, search_count')
+            .eq('id', identifier)
+            .single();
+
+          if (selectError && selectError.code !== 'PGRST116') {
+            console.error('Error checking existing business:', selectError);
+          }
+
+          // Prepare upsert data
+          const upsertData = {
+            id: identifier,
+            name: business.name,
+            phone: business.phone || null,
+            email: business.email || null,
+            address: business.address || null,
+            rating: existing ? Math.max(business.rating, existing.rating) : business.rating,
+            website: business.website || null,
+            logo: business.logo || null,
+            source: business.source || null,
+            description: business.description || null,
+            latitude: business.location?.lat || null,
+            longitude: business.location?.lng || null,
+            last_updated: new Date().toISOString(),
+            search_count: existing ? existing.search_count + 1 : 1
+          };
+
+          console.log('Upserting with data:', upsertData);
+
+          const { error: upsertError } = await this.supabase
+            .from('businesses')
+            .upsert(upsertData, {
+              onConflict: 'id'
+            });
+
+          if (upsertError) {
+            console.error('Error upserting business:', upsertError);
+            console.error('Failed business data:', upsertData);
+          } else {
+            console.log(`Successfully upserted business: ${business.name}`);
+          }
+        } catch (businessError) {
+          console.error('Error processing business:', business.name, businessError);
+        }
+      }
+    } catch (error) {
+      console.error('Error saving businesses to Supabase:', error);
+      throw error;
+    }
+  }
+} 

src/lib/supabase.ts

@@ -0,0 +1,42 @@
+import { createClient } from '@supabase/supabase-js';
+import { env } from '../config/env';
+
+// Validate Supabase configuration
+if (!env.supabase.url || !env.supabase.anonKey) {
+  throw new Error('Missing Supabase configuration');
+}
+
+// Create Supabase client
+export const supabase = createClient(
+  env.supabase.url,
+  env.supabase.anonKey,
+  {
+    auth: {
+      autoRefreshToken: true,
+      persistSession: true
+    }
+  }
+);
+
+// Test the connection on startup
+async function testConnection() {
+  try {
+    console.log('Checking Supabase connection...');
+    console.log('URL:', env.supabase.url);
+    
+    const { error } = await supabase
+      .from('businesses')
+      .select('count', { count: 'planned', head: true });
+
+    if (error) {
+      console.error('❌ Supabase initialization error:', error);
+    } else {
+      console.log('✅ Supabase connection initialized successfully');
+    }
+  } catch (error) {
+    console.error('❌ Failed to initialize Supabase:', error);
+  }
+}
+
+// Run the test
+testConnection().catch(console.error); 

src/lib/types.ts

@@ -0,0 +1,28 @@
+export interface BusinessData {
+    id?: string;
+    name: string;
+    phone?: string;
+    email?: string;
+    address?: string;
+    rating?: number;
+    website?: string;
+    logo?: string;
+    source?: string;
+    description?: string;
+    location?: {
+        lat: number;
+        lng: number;
+    };
+    latitude?: number;
+    longitude?: number;
+    place_id?: string;
+    photos?: string[];
+    openingHours?: string[];
+    distance?: {
+        value: number;
+        unit: string;
+    };
+    last_updated?: string;
+    search_count?: number;
+    created_at?: string;
+} 

src/lib/utils/cache.ts

@@ -0,0 +1,36 @@
+interface CacheItem<T> {
+  data: T;
+  timestamp: number;
+}
+
+export class Cache<T> {
+  private store = new Map<string, CacheItem<T>>();
+  private ttl: number;
+
+  constructor(ttlMinutes: number = 60) {
+    this.ttl = ttlMinutes * 60 * 1000;
+  }
+
+  set(key: string, value: T): void {
+    this.store.set(key, {
+      data: value,
+      timestamp: Date.now()
+    });
+  }
+
+  get(key: string): T | null {
+    const item = this.store.get(key);
+    if (!item) return null;
+
+    if (Date.now() - item.timestamp > this.ttl) {
+      this.store.delete(key);
+      return null;
+    }
+
+    return item.data;
+  }
+
+  clear(): void {
+    this.store.clear();
+  }
+} 

src/lib/utils/dataCleanup.ts

@@ -0,0 +1,30 @@
+export function normalizePhoneNumber(phone: string): string {
+  return phone.replace(/[^\d]/g, '');
+}
+
+export function normalizeAddress(address: string): string {
+  // Remove common suffixes and standardize format
+  return address
+    .toLowerCase()
+    .replace(/(street|st\.?|avenue|ave\.?|road|rd\.?)/g, '')
+    .trim();
+}
+
+export function extractZipCode(text: string): string | null {
+  const match = text.match(/\b\d{5}(?:-\d{4})?\b/);
+  return match ? match[0] : null;
+}
+
+export function calculateReliabilityScore(business: Business): number {
+  let score = 0;
+  
+  // More complete data = higher score
+  if (business.phone) score += 2;
+  if (business.website) score += 1;
+  if (business.email) score += 1;
+  if (business.hours) score += 2;
+  if (business.services.length > 0) score += 1;
+  if (business.reviewCount > 10) score += 2;
+  
+  return score;
+} 

src/lib/utils/rateLimiter.ts

@@ -0,0 +1,23 @@
+export class RateLimiter {
+  private timestamps: number[] = [];
+  private readonly windowMs: number;
+  private readonly maxRequests: number;
+
+  constructor(windowMs: number = 60000, maxRequests: number = 30) {
+    this.windowMs = windowMs;
+    this.maxRequests = maxRequests;
+  }
+
+  async waitForSlot(): Promise<void> {
+    const now = Date.now();
+    this.timestamps = this.timestamps.filter(time => now - time < this.windowMs);
+
+    if (this.timestamps.length >= this.maxRequests) {
+      const oldestRequest = this.timestamps[0];
+      const waitTime = this.windowMs - (now - oldestRequest);
+      await new Promise(resolve => setTimeout(resolve, waitTime));
+    }
+
+    this.timestamps.push(now);
+  }
+} 

src/lib/utils/structuredDataParser.ts

@@ -0,0 +1,119 @@
+import * as cheerio from 'cheerio';
+
+interface StructuredData {
+  name?: string;
+  email?: string;
+  phone?: string;
+  address?: string;
+  socialProfiles?: string[];
+  openingHours?: Record<string, string>;
+  description?: string;
+}
+
+export class StructuredDataParser {
+  static parse($: cheerio.CheerioAPI): StructuredData[] {
+    const results: StructuredData[] = [];
+
+    // Parse JSON-LD
+    $('script[type="application/ld+json"]').each((_, element) => {
+      try {
+        const data = JSON.parse($(element).html() || '{}');
+        if (Array.isArray(data)) {
+          data.forEach(item => this.parseStructuredItem(item, results));
+        } else {
+          this.parseStructuredItem(data, results);
+        }
+      } catch (e) {
+        console.error('Error parsing JSON-LD:', e);
+      }
+    });
+
+    // Parse microdata
+    $('[itemtype]').each((_, element) => {
+      const type = $(element).attr('itemtype');
+      if (type?.includes('Organization') || type?.includes('LocalBusiness')) {
+        const data: StructuredData = {
+          name: $('[itemprop="name"]', element).text(),
+          email: $('[itemprop="email"]', element).text(),
+          phone: $('[itemprop="telephone"]', element).text(),
+          address: this.extractMicrodataAddress($, element),
+          socialProfiles: this.extractSocialProfiles($, element)
+        };
+        results.push(data);
+      }
+    });
+
+    // Parse RDFa
+    $('[typeof="Organization"], [typeof="LocalBusiness"]').each((_, element) => {
+      const data: StructuredData = {
+        name: $('[property="name"]', element).text(),
+        email: $('[property="email"]', element).text(),
+        phone: $('[property="telephone"]', element).text(),
+        address: this.extractRdfaAddress($, element),
+        socialProfiles: this.extractSocialProfiles($, element)
+      };
+      results.push(data);
+    });
+
+    return results;
+  }
+
+  private static parseStructuredItem(data: any, results: StructuredData[]): void {
+    if (data['@type'] === 'Organization' || data['@type'] === 'LocalBusiness') {
+      results.push({
+        name: data.name,
+        email: data.email,
+        phone: data.telephone,
+        address: this.formatAddress(data.address),
+        socialProfiles: this.extractSocialUrls(data),
+        openingHours: this.parseOpeningHours(data.openingHours),
+        description: data.description
+      });
+    }
+  }
+
+  private static formatAddress(address: any): string | undefined {
+    if (typeof address === 'string') return address;
+    if (typeof address === 'object') {
+      const parts = [
+        address.streetAddress,
+        address.addressLocality,
+        address.addressRegion,
+        address.postalCode,
+        address.addressCountry
+      ].filter(Boolean);
+      return parts.join(', ');
+    }
+    return undefined;
+  }
+
+  private static extractSocialUrls(data: any): string[] {
+    const urls: string[] = [];
+    if (data.sameAs) {
+      if (Array.isArray(data.sameAs)) {
+        urls.push(...data.sameAs);
+      } else if (typeof data.sameAs === 'string') {
+        urls.push(data.sameAs);
+      }
+    }
+    return urls;
+  }
+
+  private static parseOpeningHours(hours: any): Record<string, string> | undefined {
+    if (!hours) return undefined;
+    
+    if (Array.isArray(hours)) {
+      const schedule: Record<string, string> = {};
+      hours.forEach(spec => {
+        const match = spec.match(/^(\w+)(-\w+)?\s+(\d\d:\d\d)-(\d\d:\d\d)$/);
+        if (match) {
+          schedule[match[1]] = `${match[3]}-${match[4]}`;
+        }
+      });
+      return schedule;
+    }
+    return undefined;
+  }
+
+  // ... helper methods for microdata and RDFa parsing ...
+} 

src/routes/api.ts

@@ -0,0 +1,88 @@
+import { Router } from 'express';
+import { searchBusinesses } from '../lib/searxng';
+import { categories } from '../lib/categories';
+import { supabase } from '../lib/supabase';
+import { BusinessData } from '../lib/types';
+
+const router = Router();
+
+// Categories endpoint
+router.get('/categories', (req, res) => {
+  res.json(categories);
+});
+
+// Search endpoint
+router.get('/search', async (req, res) => {
+    try {
+        const query = req.query.q as string;
+        const [searchTerm, location] = query.split(' in ');
+        
+        if (!query) {
+            return res.status(400).json({ error: 'Search query is required' });
+        }
+
+        // Set headers for streaming response
+        res.setHeader('Content-Type', 'application/json');
+        res.setHeader('Transfer-Encoding', 'chunked');
+
+        // First, search in Supabase
+        const { data: existingResults, error: dbError } = await supabase
+            .from('businesses')
+            .select('*')
+            .or(`name.ilike.%${searchTerm}%, description.ilike.%${searchTerm}%`)
+            .ilike('address', `%${location}%`);
+
+        if (dbError) {
+            console.error('Supabase search error:', dbError);
+        }
+
+        // Send existing results immediately if there are any
+        if (existingResults && existingResults.length > 0) {
+            const chunk = JSON.stringify({ 
+                source: 'database',
+                results: existingResults 
+            }) + '\n';
+            res.write(chunk);
+        }
+
+        // Start background search
+        const searchPromise = searchBusinesses(query, {
+            onProgress: (status, progress) => {
+                const chunk = JSON.stringify({ 
+                    source: 'search',
+                    status,
+                    progress,
+                }) + '\n';
+                res.write(chunk);
+            }
+        });
+
+        const results = await searchPromise;
+        
+        // Send final results
+        const finalChunk = JSON.stringify({ 
+            source: 'search',
+            results,
+            complete: true 
+        }) + '\n';
+        res.write(finalChunk);
+        res.end();
+
+    } catch (error: unknown) {
+        console.error('Search error:', error);
+        const errorResponse = { 
+            error: 'An error occurred while searching',
+            details: error instanceof Error ? error.message : 'Unknown error'
+        };
+        
+        // Only send error response if headers haven't been sent
+        if (!res.headersSent) {
+            res.status(500).json(errorResponse);
+        } else {
+            res.write(JSON.stringify(errorResponse));
+            res.end();
+        }
+    }
+});
+
+export default router; 

src/test-supabase.ts

@@ -0,0 +1,102 @@
+import { createClient } from '@supabase/supabase-js';
+import dotenv from 'dotenv';
+
+// Load environment variables
+dotenv.config();
+
+async function testSupabaseConnection() {
+    console.log('Testing Supabase connection...');
+    console.log('URL:', process.env.SUPABASE_URL);
+    console.log('Key length:', process.env.SUPABASE_KEY?.length || 0);
+
+    try {
+        const supabase = createClient(
+            process.env.SUPABASE_URL!,
+            process.env.SUPABASE_KEY!,
+            {
+                auth: {
+                    autoRefreshToken: true,
+                    persistSession: true
+                }
+            }
+        );
+
+        // Test businesses table
+        console.log('\nTesting businesses table:');
+        const testBusiness = {
+            id: 'test_' + Date.now(),
+            name: 'Test Business',
+            phone: '123-456-7890',
+            email: 'test@example.com',
+            address: '123 Test St',
+            rating: 5,
+            website: 'https://test.com',
+            source: 'test',
+            description: 'Test description',
+            latitude: 39.7392,
+            longitude: -104.9903,
+            search_count: 1,
+            created_at: new Date().toISOString()
+        };
+
+        const { error: insertBusinessError } = await supabase
+            .from('businesses')
+            .insert([testBusiness])
+            .select();
+
+        if (insertBusinessError) {
+            console.error('❌ INSERT business error:', insertBusinessError);
+        } else {
+            console.log('✅ INSERT business OK');
+            // Clean up
+            await supabase.from('businesses').delete().eq('id', testBusiness.id);
+        }
+
+        // Test searches table
+        console.log('\nTesting searches table:');
+        const testSearch = {
+            query: 'test query',
+            location: 'test location',
+            results_count: 0,
+            timestamp: new Date().toISOString()
+        };
+
+        const { error: insertSearchError } = await supabase
+            .from('searches')
+            .insert([testSearch])
+            .select();
+
+        if (insertSearchError) {
+            console.error('❌ INSERT search error:', insertSearchError);
+        } else {
+            console.log('✅ INSERT search OK');
+        }
+
+        // Test cache table
+        console.log('\nTesting cache table:');
+        const testCache = {
+            key: 'test_key_' + Date.now(),
+            value: { test: true },
+            created_at: new Date().toISOString(),
+            expires_at: new Date(Date.now() + 3600000).toISOString()
+        };
+
+        const { error: insertCacheError } = await supabase
+            .from('cache')
+            .insert([testCache])
+            .select();
+
+        if (insertCacheError) {
+            console.error('❌ INSERT cache error:', insertCacheError);
+        } else {
+            console.log('✅ INSERT cache OK');
+            // Clean up
+            await supabase.from('cache').delete().eq('key', testCache.key);
+        }
+
+    } catch (error: any) {
+        console.error('❌ Unexpected error:', error);
+    }
+}
+
+testSupabaseConnection().catch(console.error); 

src/tests/supabaseTest.ts

@@ -0,0 +1,94 @@
+import '../config/env';  // Load env vars first
+import { CacheService } from '../lib/services/cacheService';
+import type { PostgrestError } from '@supabase/supabase-js';
+import { env } from '../config/env';
+
+async function testSupabaseConnection() {
+  console.log('\n🔍 Testing Supabase Connection...');
+  console.log('Using Supabase URL:', env.supabase.url);
+  
+  try {
+    // Test data
+    const testData = {
+      category: 'test_category',
+      location: 'test_location',
+      results: [{
+        name: 'Test Business',
+        phone: '123-456-7890',
+        email: 'test@example.com',
+        address: '123 Test St, Test City, TS 12345',
+        rating: 95,
+        website: 'https://test.com',
+        logo: '',
+        source: 'test',
+        description: 'Test business description'
+      }]
+    };
+
+    console.log('\n1️⃣ Testing write operation...');
+    await CacheService.cacheResults(
+      testData.category,
+      testData.location,
+      testData.results,
+      env.cache.durationDays
+    );
+    console.log('✅ Write successful');
+
+    console.log('\n2️⃣ Testing read operation...');
+    const cachedResults = await CacheService.getCachedResults(
+      testData.category,
+      testData.location
+    );
+    
+    if (cachedResults && cachedResults.length > 0) {
+      console.log('✅ Read successful');
+      console.log('\nCached data:', JSON.stringify(cachedResults[0], null, 2));
+    } else {
+      throw new Error('No results found in cache');
+    }
+
+    console.log('\n3️⃣ Testing update operation...');
+    const updatedResults = [...testData.results];
+    updatedResults[0].rating = 98;
+    await CacheService.updateCache(
+      testData.category,
+      testData.location,
+      updatedResults
+    );
+    console.log('✅ Update successful');
+
+    console.log('\n✨ All tests passed! Supabase connection is working properly.\n');
+
+  } catch (error: unknown) {
+    console.error('\n❌ Test failed:');
+    
+    if (error instanceof Error) {
+      console.error('Error message:', error.message);
+      
+      // Check if it's a Supabase error by looking at the shape of the error object
+      const isSupabaseError = (err: any): err is PostgrestError => 
+        'code' in err && 'details' in err && 'hint' in err && 'message' in err;
+      
+      if (error.message.includes('connection') || isSupabaseError(error)) {
+        console.log('\n📋 Troubleshooting steps:');
+        console.log('1. Check if your SUPABASE_URL and SUPABASE_ANON_KEY are correct in .env');
+        console.log('2. Verify that the search_cache table exists in your Supabase project');
+        console.log('3. Check if RLS policies are properly configured');
+        
+        if (isSupabaseError(error)) {
+          console.log('\nSupabase error details:');
+          console.log('Code:', error.code);
+          console.log('Details:', error.details);
+          console.log('Hint:', error.hint);
+        }
+      }
+    } else {
+      console.error('Unknown error:', error);
+    }
+    
+    process.exit(1);
+  }
+}
+
+// Run the test
+testSupabaseConnection(); 

src/tests/testSearch.ts

@@ -0,0 +1,26 @@
+import { searchSearxng } from '../lib/searxng';
+
+async function testSearchEngine() {
+  try {
+    console.log('Testing SearxNG connection...');
+    
+    const results = await searchSearxng('plumbers in Denver', {
+      engines: ['google', 'bing', 'duckduckgo'],
+      pageno: 1
+    });
+
+    if (results && results.results && results.results.length > 0) {
+      console.log('✅ Search successful!');
+      console.log('Number of results:', results.results.length);
+      console.log('First result:', results.results[0]);
+    } else {
+      console.log('❌ No results found');
+    }
+
+  } catch (error) {
+    console.error('❌ Search test failed:', error);
+    console.error('Make sure SearxNG is running on http://localhost:4000');
+  }
+}
+
+testSearchEngine(); 

src/types/business.ts

@@ -0,0 +1,28 @@
+export interface Business {
+  id: string;
+  name: string;
+  phone: string;
+  address: string;
+  city: string;
+  state: string;
+  zip: string;
+  category: string[];
+  rating: number;
+  reviewCount: number;
+  license?: string;
+  services: string[];
+  hours: Record<string, string>;
+  website?: string;
+  email?: string;
+  verified: boolean;
+  lastUpdated: Date;
+}
+
+export interface SearchParams {
+  location: string;
+  category?: string;
+  radius?: number;
+  minRating?: number;
+  sortBy?: 'rating' | 'distance' | 'reviewCount';
+  verified?: boolean;
+}