Scraper d'Annonces Médicales Multi-Sources

1const functions = require("firebase-functions");
2const admin = require("firebase-admin");
3const axios = require("axios");
4const { v4: uuidv4 } = require("uuid");
5
6admin.initializeApp();
7const db = admin.firestore();
8
9exports.fetchJobAds2 = functions
10  .pubsub
11  .schedule("every 24 hours") // Cron quotidien
12  .onRun(async (context) => {
13    const apiUrl = process.env.PARTNER_API_URL;
14
15    try {
16      // 1. Requête API externe
17      const response = await axios.get(apiUrl, {
18        timeout: 30000 // 30s timeout
19      });
20      const jobAds = response.data.ads;
21      const count = response.data.count;
22
23      console.log(`[SCRAPER] Fetched ${jobAds.length} ads from API`);
24
25      // 2. Récupérer annonces et cliniques existantes
26      const existingAdsSnapshot = await db.collection("annonces").get();
27      const existingClinicsSnapshot = await db.collection("cliniques").get();
28
29      const existingAdsMap = new Map();
30      const existingClinicsMap = new Map();
31
32      existingAdsSnapshot.forEach((doc) => {
33        existingAdsMap.set(doc.id, doc.data());
34      });
35      existingClinicsSnapshot.forEach((doc) => {
36        existingClinicsMap.set(doc.id, doc.data());
37      });
38
39      const batch = db.batch();
40      const currentAdReferences = new Set();
41      const currentClinicNames = new Set();
42
43      let adsCreated = 0;
44      let adsUpdated = 0;
45      let clinicsCreated = 0;
46
47      // 3. Traiter chaque annonce
48      for (const ad of jobAds) {
49        // Générer référence unique si manquante
50        let reference = ad.reference || uuidv4();
51        const existingAd = existingAdsMap.get(reference);
52        currentAdReferences.add(reference);
53
54        // Normaliser les données
55        const normalizedAd = normalizeJobAd(ad);
56
57        // Créer ou mettre à jour
58        if (!existingAd) {
59          batch.set(db.collection("annonces").doc(reference), normalizedAd);
60          adsCreated++;
61        } else if (hasChanges(existingAd, normalizedAd)) {
62          batch.update(db.collection("annonces").doc(reference), normalizedAd);
63          adsUpdated++;
64        }
65
66        // Gérer la clinique associée
67        const clinicName = ad.clinic_name;
68        if (clinicName && !existingClinicsMap.has(clinicName)) {
69          const clinicData = extractClinicData(ad);
70          batch.set(db.collection("cliniques").doc(clinicName), clinicData);
71          clinicsCreated++;
72        }
73      }
74
75      // 4. Supprimer annonces obsolètes
76      const adsDeleted = [];
77      for (const [ref, data] of existingAdsMap) {
78        if (!currentAdReferences.has(ref)) {
79          await deleteAnnonceWithProspects(db.collection("annonces").doc(ref));
80          adsDeleted.push(ref);
81        }
82      }
83
84      // 5. Commit batch
85      await batch.commit();
86
87      // 6. Synchroniser Algolia
88      await syncAlgoliaIndex(jobAds);
89
90      console.log(`[SCRAPER] Summary:
91        - Created: ${adsCreated}
92        - Updated: ${adsUpdated}
93        - Deleted: ${adsDeleted.length}
94        - Clinics created: ${clinicsCreated}
95      `);
96
97      return {
98        success: true,
99        stats: { adsCreated, adsUpdated, adsDeleted: adsDeleted.length }
100      };
101
102    } catch (error) {
103      console.error("[SCRAPER] Error:", error);
104      // Envoyer alerte email
105      await sendErrorAlert(error);
106      throw error;
107    }
108  });
109