Server : Apache/2 System : Linux server-15-235-50-60 5.15.0-164-generic #174-Ubuntu SMP Fri Nov 14 20:25:16 UTC 2025 x86_64 User : gositeme ( 1004) PHP Version : 8.2.29 Disable Function : exec,system,passthru,shell_exec,proc_close,proc_open,dl,popen,show_source,posix_kill,posix_mkfifo,posix_getpwuid,posix_setpgid,posix_setsid,posix_setuid,posix_setgid,posix_seteuid,posix_setegid,posix_uname Directory :  /home/gositeme/domains/lavocat.quebec/public_html/scripts/

Upload File : current_dir [ Writeable ] document_root [ Writeable ]

const { BarreauScraper } = require('../src/lib/barreau-scraper.js');

async function testBarreauScraper() {
  console.log('🧪 Testing Barreau Scraper...\n');

  const scraper = new BarreauScraper();

  try {
    // Initialize the scraper
    console.log('1️⃣ Initializing scraper...');
    await scraper.initialize();
    console.log('✅ Scraper initialized successfully\n');

    // Test with a small sample (just first page)
    console.log('2️⃣ Testing with first page only...');
    
    // Override the scrapeEntireDirectory method for testing
    const originalMethod = scraper.scrapeEntireDirectory.bind(scraper);
    scraper.scrapeEntireDirectory = async function() {
      console.log('🔍 Starting test scraping (first page only)...');
      
      if (!this.browser || !this.page) {
        throw new Error('Scraper not initialized');
      }

      this.isRunning = true;
      this.progress = {
        totalPages: 1, // Force to 1 for testing
        currentPage: 0,
        totalLawyers: 0,
        importedLawyers: 0,
        errors: 0,
        startTime: Date.now()
      };

      try {
        // Navigate to the main directory page
        await this.page.goto('https://www.barreau.qc.ca/fr/trouver-avocat', {
          waitUntil: 'networkidle2',
          timeout: 30000
        });

        console.log('📄 Scraping page 1/1');
        
        // Wait for lawyer listings to load
        await this.page.waitForSelector('.lawyer-listing, .avocat-item, .result-item, .search-result', { 
          timeout: 10000 
        });

        // Get all lawyer links on this page
        const lawyerLinks = await this.page.evaluate(() => {
          const links = Array.from(document.querySelectorAll('a[href*="/avocat/"], a[href*="/lawyer/"], .lawyer-listing a, .avocat-item a, .search-result a'));
          return links.map(link => link.href).filter(href => 
            href.includes('/avocat/') || 
            href.includes('/lawyer/') ||
            href.includes('/repertoire/')
          );
        });

        console.log(`📋 Found ${lawyerLinks.length} lawyers on page 1`);

        // Test with first 3 lawyers only
        const testLawyers = lawyerLinks.slice(0, 3);
        console.log(`🧪 Testing with ${testLawyers.length} lawyers...`);

        for (const lawyerUrl of testLawyers) {
          try {
            console.log(`🔍 Scraping: ${lawyerUrl}`);
            await this.scrapeLawyerProfile(lawyerUrl);
            this.progress.importedLawyers++;
            console.log(`✅ Successfully scraped lawyer`);
            
            // Add delay between lawyer profiles
            await this.delay(2000);
            
          } catch (error) {
            console.error(`❌ Error scraping lawyer profile ${lawyerUrl}:`, error.message);
            this.progress.errors++;
          }
        }

        console.log('🎉 Test scraping completed!');
        this.printProgressReport();

      } catch (error) {
        console.error('❌ Fatal error during test scraping:', error);
        throw error;
      } finally {
        this.isRunning = false;
      }
    };

    // Run the test
    await scraper.scrapeEntireDirectory();

    console.log('\n🎯 Test Results:');
    console.log('================');
    const progress = scraper.getProgress();
    console.log(`📊 Lawyers imported: ${progress.importedLawyers}`);
    console.log(`❌ Errors: ${progress.errors}`);
    console.log(`📈 Success rate: ${((progress.importedLawyers / (progress.importedLawyers + progress.errors)) * 100).toFixed(1)}%`);

  } catch (error) {
    console.error('❌ Test failed:', error);
  } finally {
    // Clean up
    console.log('\n🧹 Cleaning up...');
    await scraper.stop();
    console.log('✅ Test completed');
  }
}

// Run the test
testBarreauScraper().catch(console.error);