Merge pull request #7 from renomureza/fix-tafsir

perbaikan tafsir kemenag & typo key revelation

README.md

@@ -65,7 +65,7 @@ Berikut beberapa sumber data yang digunakan:
 
 - `yarn start` - menjalankan server.
 - `yarn dev` - menjalankan server pengembangan.
-- `yarn build:data` - untuk membangun ulang data quran utama.
+- `yarn build:quran` - untuk membangun ulang data quran utama.
 
 ## License
 

data/scripts/quranBuilder.js

@@ -1,9 +1,10 @@
-const fs = require("fs");
+const fs = require("fs/promises");
 const { default: axios } = require("axios");
-const { data: quranKemenag } = require("../tmp/quran-kemenag.json");
 const tafsirQuraish = require("../tmp/tafsir-quraish.json");
 const tafsirJalalayn = require("../tmp/tafsir-jalalayn.json");
 const { pathToData, cleanHtmlTag } = require("../../utils/utility");
+const tafsirKemenag = require("../../data/tmp/tafsir-kemenag.json");
+const surahKemenag = require("../../data/tmp/surah-kemenag.json");
 
 const httpToHttps = (str = "") => str.replace("http", "https");
 
@@ -15,6 +16,25 @@ const getTafsirJalalayn = (numberSurah, numberAyah) => {
   return tafsirJalalayn[`${numberSurah}.${numberAyah}`];
 };
 
+const getTafsirKemenag = (ayahIdx) => {
+  return {
+    short: cleanHtmlTag(tafsirKemenag[ayahIdx].short.text),
+    long:
+      ayahIdx !== 293
+        ? removeWiredQuestionMark(
+            cleanHtmlTag(tafsirKemenag[ayahIdx].long.text)
+          )
+        : cleanHtmlTag(tafsirKemenag[ayahIdx].long.text),
+  };
+};
+
+const removeWiredQuestionMark = (tafsir) => {
+  // https://quran.kemenag.go.id/api/v1/tafsirbyayat/293
+  return tafsir
+    .replace(/\n+[ |():|\?]+\n/g, "")
+    .replace(/\u2026[ \.|\.]+/g, "");
+};
+
 const getSurahsQuranCloud = () => {
   return axios
     .get("https://api.alquran.cloud/v1/quran/quran-uthmani")
@@ -27,7 +47,7 @@ const getAudioAndDescriptionSurahs = () => {
     .then(({ data }) => data);
 };
 
-const getAudioPerAyah = (numberAyahInQuran) => {
+const getAudioAyah = (numberAyahInQuran) => {
   return {
     alafasy: `https://cdn.islamic.network/quran/audio/128/ar.alafasy/${numberAyahInQuran}.mp3`,
     ahmedajamy: `https://cdn.islamic.network/quran/audio/128/ar.ahmedajamy/${numberAyahInQuran}.mp3`,
@@ -46,11 +66,11 @@ const getImageAyah = (numberSurah, numberAyah) => {
 };
 
 const getBismillah = () => {
-  const { aya_text, translation_aya_text } = quranKemenag[0].verses[0];
+  const { aya_text, translation_aya_text } = surahKemenag[0].ayahs[0];
   return {
     arab: aya_text,
     translation: translation_aya_text,
-    audio: getAudioPerAyah(1),
+    audio: getAudioAyah(1),
   };
 };
 
@@ -83,13 +103,10 @@ const ayahTransformer = (numberSurah, surahsFromQuranCloud) => {
     },
     arab: ayah.aya_text,
     translation: ayah.translation_aya_text,
-    audio: getAudioPerAyah(ayah.aya_id),
+    audio: getAudioAyah(ayah.aya_id),
     image: getImageAyah(numberSurah, ayah.aya_number),
     tafsir: {
-      kemenag: {
-        short: ayah.tafsir.short.text,
-        long: ayah.tafsir.long.text,
-      },
+      kemenag: getTafsirKemenag(ayah.aya_id - 1),
       quraish: getTafsirQuraish(numberSurah, ayah.aya_number),
       jalalayn: getTafsirJalalayn(numberSurah, ayah.aya_number),
     },
@@ -103,25 +120,27 @@ const surahTransformer = (surahsQuranCloud, surahsAudioAndDescription) => {
     numberOfAyahs: surah.count_ayat,
     name: surah.surat_name,
     translation: surah.surat_terjemahan,
-    revalation: revelationEnToId(surahsQuranCloud[surahIdx].revelationType),
+    revelation: revelationEnToId(surahsQuranCloud[surahIdx].revelationType),
     description: cleanHtmlTag(surahsAudioAndDescription[surahIdx].keterangan),
     audio: httpToHttps(surahsAudioAndDescription[surahIdx].audio),
     bismillah: getBismillah(),
-    ayahs: surah.verses.map(ayahTransformer(surah.id, surahsQuranCloud)),
+    ayahs: surah.ayahs.map(ayahTransformer(surah.id, surahsQuranCloud)),
   });
 };
 
 const quranDataBuilder = async () => {
   const surahsQuranCloud = await getSurahsQuranCloud();
   const surahsAudioAndDescription = await getAudioAndDescriptionSurahs();
-  return quranKemenag.map(
+  return surahKemenag.map(
     surahTransformer(surahsQuranCloud, surahsAudioAndDescription)
   );
 };
 
 const main = async () => {
+  console.log("START building");
   const quran = await quranDataBuilder();
-  fs.writeFileSync(pathToData("quran.json"), JSON.stringify(quran));
+  await fs.writeFile(pathToData("quran.json"), JSON.stringify(quran));
+  console.log("FINISHED building");
 };
 
 main();

data/scripts/surahKemenagScraper.js

@@ -0,0 +1,48 @@
+const fs = require("fs/promises");
+const { default: axios } = require("axios");
+const {
+  delay,
+  logSuccess,
+  logError,
+  logInfo,
+  pathToData,
+} = require("../../utils/utility");
+
+const getListSurah = () => {
+  return axios
+    .get("https://quran.kemenag.go.id/api/v1/surat/0/114")
+    .then((res) => res.data.data);
+};
+
+const getAyahs = async (numberSurah) => {
+  try {
+    await delay(1);
+    const verse = await axios.get(
+      `https://quran.kemenag.go.id/api/v1/ayatweb/${numberSurah}/0/0/10000/`
+    );
+    logSuccess(`AYAHS: surah ${numberSurah} SUCCESS!`);
+    return verse.data.data;
+  } catch (error) {
+    logError(`AYAHS: surah ${numberSurah} FAIL!`);
+    logInfo(`AYAHS: surah ${numberSurah} RE-CRAWL!`);
+    await delay(2);
+    return await getAyahs(numberSurah);
+  }
+};
+
+const main = async () => {
+  const surahs = await getListSurah();
+
+  for (const surah of surahs) {
+    const ayahs = await getAyahs(surah.id);
+    surahs[surah.id - 1].ayahs = ayahs;
+  }
+
+  await fs.writeFile(
+    pathToData("tmp", "kemenag", "surah.json"),
+    JSON.stringify(surahs)
+  );
+  logSuccess("Everything is DONE!");
+};
+
+main();

data/scripts/tafsirKemenagScraper.js

@@ -0,0 +1,41 @@
+const fs = require("fs/promises");
+const { default: axios } = require("axios");
+const {
+  logSuccess,
+  logInfo,
+  logError,
+  delay,
+  pathToData,
+} = require("../../utils/utility");
+
+const getTafsir = async (numberAyah) => {
+  try {
+    await delay(2);
+    const tafsir = await axios.get(
+      `https://quran.kemenag.go.id/api/v1/tafsirbyayat/${numberAyah}`
+    );
+    logSuccess(`TAFSIR: ayah ${numberAyah} SUCCESS!`);
+    return { short: tafsir.data.tafsir[0], long: tafsir.data.tafsir[1] };
+  } catch (error) {
+    logError(`TAFSIR: ayah ${numberAyah} FAIL!`);
+    logInfo(`TAFSIR: ayah ${numberAyah} RE-CRAWL!`);
+    await delay(2);
+    return await getTafsir(numberAyah);
+  }
+};
+
+const main = async () => {
+  const tafsirs = [];
+
+  for (let i = 1; i <= 6236; i++) {
+    const tafsir = await getTafsir(i);
+    tafsirs.push(tafsir);
+  }
+
+  await fs.writeFile(
+    pathToData("tmp", "kemenag", "tafsir.json"),
+    JSON.stringify(tafsirs)
+  );
+};
+
+main();