test_comprehensive_scraper.py

#!/usr/bin/env python3
"""
Comprehensive LinkedIn Scraper Test
Tests all scenarios: login, redirects, security verification, etc.
"""

import os
import time
from dotenv import load_dotenv
from scraper_authenticated import scrape_linkedin_authenticated

load_dotenv()

def test_comprehensive_scenarios():
    """Test all scenarios with the enhanced scraper"""
    
    print("🧪 Comprehensive LinkedIn Scraper Test")
    print("=" * 60)
    
    # Test scenarios
    test_cases = [
        {
            "name": "Public Profile - Ankit Yadav",
            "url": "https://www.linkedin.com/in/liveankit",
            "expected": "Should work with automatic login"
        },
        {
            "name": "High-Profile - Bill Gates",
            "url": "https://www.linkedin.com/in/williamhgates/",
            "expected": "Should work with automatic login"
        },
        {
            "name": "CEO Profile - Satya Nadella",
            "url": "https://www.linkedin.com/in/satyanadella/",
            "expected": "Should work with automatic login"
        },
        {
            "name": "Student Profile - Hiren Danecha",
            "url": "https://in.linkedin.com/in/hiren-danecha-695a51110",
            "expected": "Should work with automatic login"
        }
    ]
    
    results = []
    
    for i, test_case in enumerate(test_cases, 1):
        print(f"\n{'='*20} Test {i}: {test_case['name']} {'='*20}")
        print(f"🔗 URL: {test_case['url']}")
        print(f"💡 Expected: {test_case['expected']}")
        print("-" * 60)
        
        try:
            start_time = time.time()
            result = scrape_linkedin_authenticated(test_case['url'])
            end_time = time.time()
            
            duration = end_time - start_time
            
            # Analyze result
            success = result.get('full_name') and result.get('full_name') not in [
                'Profile Access Restricted', 
                'Profile Not Accessible',
                'Profile Extraction Failed',
                'Security Verification Error'
            ]
            
            if success:
                print(f"✅ SUCCESS! ({duration:.1f}s)")
                print(f"👤 Name: {result.get('full_name', 'N/A')}")
                print(f"💼 Headline: {result.get('headline', 'N/A')[:80]}...")
                print(f"📝 Summary: {result.get('summary', 'N/A')[:100]}...")
                
                # Check for special flags
                if result.get('security_blocked'):
                    print("⚠️ Security blocked but data extracted")
                if result.get('public_only'):
                    print("ℹ️ Public profile data only")
                if result.get('auth_required'):
                    print("⚠️ Authentication required")
                    
            else:
                print(f"❌ FAILED! ({duration:.1f}s)")
                print(f"🚫 Error: {result.get('full_name', 'Unknown error')}")
                print(f"📄 Details: {result.get('summary', 'No details')}")
            
            results.append({
                "test": test_case['name'],
                "success": success,
                "duration": duration,
                "result": result
            })
            
        except Exception as e:
            print(f"❌ EXCEPTION! ({time.time() - start_time:.1f}s)")
            print(f"💥 Error: {str(e)}")
            results.append({
                "test": test_case['name'],
                "success": False,
                "duration": time.time() - start_time,
                "error": str(e)
            })
        
        # Wait between tests
        if i < len(test_cases):
            print("\n⏳ Waiting 5 seconds before next test...")
            time.sleep(5)
    
    # Summary
    print(f"\n{'='*60}")
    print("📊 TEST SUMMARY")
    print("=" * 60)
    
    successful = sum(1 for r in results if r['success'])
    total = len(results)
    avg_duration = sum(r['duration'] for r in results) / len(results)
    
    print(f"✅ Successful: {successful}/{total} ({successful/total*100:.1f}%)")
    print(f"⏱️ Average duration: {avg_duration:.1f}s")
    print(f"❌ Failed: {total - successful}/{total}")
    
    print("\n📋 Detailed Results:")
    for result in results:
        status = "✅ PASS" if result['success'] else "❌ FAIL"
        print(f"  {status} {result['test']} ({result['duration']:.1f}s)")
        if not result['success'] and 'error' in result:
            print(f"    💥 {result['error']}")
    
    return results

def test_login_scenarios():
    """Test specific login scenarios"""
    
    print("\n🔐 Testing Login Scenarios")
    print("=" * 40)
    
    # Test 1: Direct login
    print("\n1️⃣ Testing direct login...")
    try:
        result = scrape_linkedin_authenticated("https://www.linkedin.com/in/liveankit")
        if result.get('full_name') and result.get('full_name') != 'Profile Access Restricted':
            print("✅ Direct login successful")
        else:
            print("❌ Direct login failed")
    except Exception as e:
        print(f"❌ Direct login exception: {e}")
    
    # Test 2: Profile that might redirect to login
    print("\n2️⃣ Testing profile that might redirect to login...")
    try:
        result = scrape_linkedin_authenticated("https://www.linkedin.com/in/williamhgates/")
        if result.get('full_name') and result.get('full_name') != 'Profile Access Restricted':
            print("✅ Redirect login successful")
        else:
            print("❌ Redirect login failed")
    except Exception as e:
        print(f"❌ Redirect login exception: {e}")

if __name__ == "__main__":
    print("🚀 LinkedIn Scraper Comprehensive Test Suite")
    print("=" * 60)
    
    # Check credentials
    LINKEDIN_EMAIL = os.environ.get("LINKEDIN_EMAIL")
    LINKEDIN_PASSWORD = os.environ.get("LINKEDIN_PASSWORD")
    
    if not LINKEDIN_EMAIL or not LINKEDIN_PASSWORD:
        print("❌ LinkedIn credentials not found!")
        print("Please add LINKEDIN_EMAIL and LINKEDIN_PASSWORD to your .env file")
        exit(1)
    
    print(f"📧 Email: {LINKEDIN_EMAIL}")
    print(f"🔑 Password: {'*' * len(LINKEDIN_PASSWORD)}")
    print("=" * 60)
    
    # Run comprehensive tests
    results = test_comprehensive_scenarios()
    
    # Run login scenario tests
    test_login_scenarios()
    
    print(f"\n🎉 Test suite completed!")
    print("💡 The scraper should now handle all scenarios automatically:")
    print("   ✅ Automatic login when needed")
    print("   ✅ Credential filling on login pages")
    print("   ✅ Security verification handling")
    print("   ✅ Multiple retry attempts")
    print("   ✅ Fallback methods")