experiments.json

{
  "defaults": {
    "construct_training_data": {
      "method": [0],
      "nwords": [125],
      "section": ["AR"]
    },
    "fit_clinicalbert": {
      "max-length": [-1],
      "batch-size": [-1],
      "epochs": [25],
      "learning-rate": [1e-6],
      "ifexists": ["quit"],
      "network": ["Bio_ClinicalBERT"],
      "refsource": ["all"],
      "split-method": ["24"]
    },
    "analyze_results": {
      "skip-train": [true],
      "network": ["Bio_ClinicalBERT"]
    },
    "compile_results": {
      "group-function": ["mean"]
    }
  },

  "experiments":{
    "0": {
      "name": "Compare Example Construction Methods, ClinicalBERT",
      "description": "We have six different methods of creating the example strings based on whether or not we replace the adverse event term with a common term 'EVENT', prepend the adverse event term to the beginning of the example string, or replace the event with a nonsense (unmappable) term. These different methods are built using the construct_training_data.py script using the method argument to choose between the five (0, 1, 2, 3, 4, 5, or 12). See construct_training_data.py for more detials.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": [
          "Substitute Term, Prepend Term",
          "Prepend Term Only",
          "Substitute Term Only",
          "No Changes",
          "Subsitute with Nonsense",
          "Random Sentences",
          "Substitute Term, Prepend Term, Prepend Source"
        ]
      },
      "construct_training_data": {
        "method": [0, 1, 2, 3, 4, 5, 12],
        "nwords": [30],
        "section": ["AR"]
      },
      "fit_clinicalbert": {
        "max-length": [32]
      }
    },

    "0A": {
      "name": "Compare Example Construction Methods, PubMedBERT",
      "description": "We have six different methods of creating the example strings based on whether or not we replace the adverse event term with a common term 'EVENT', prepend the adverse event term to the beginning of the example string, or replace the event with a nonsense (unmappable) term. These different methods are built using the construct_training_data.py script using the method argument to choose between the five (0, 1, 2, 3, 4, 5, or 12). See construct_training_data.py for more detials.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": [
          "Substitute Term, Prepend Term",
          "Prepend Term Only",
          "Substitute Term Only",
          "No Changes",
          "Subsitute with Nonsense",
          "Random Sentences",
          "Substitute Term, Prepend Term, Prepend Source"
        ]
      },
      "construct_training_data": {
        "method": [0, 1, 2, 3, 4, 5, 12],
        "nwords": [30],
        "section": ["AR"]
      },
      "fit_clinicalbert": {
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"],
        "max-length": [32]
      }
    },

    "1": {
      "name": "Compare PubMedBERT to ClinicalBERT",
      "description": "PubMedBERT is a new neural network made available from microsoft. We'd like to know how it compares to using ClinicalBERT for our task.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "network",
        "labels": [
          "ClinicalBERT",
          "PubMedBERT"
        ]
      },
      "construct_training_data": {
        "nwords": [60]
      },
      "fit_clinicalbert": {
        "network": ["Bio_ClinicalBERT", "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"],
        "epochs": [20]
      }
    },

    "2": {
      "name": "Compare values for nwords",
      "description": "The number of words we extract from the labels and use to make the examples is a core component of the reference set. Essentially the thinking is the more words the better, however, that also exponentially increases the running time. In this experiment we adjust the number of words used in the example strings from 30 up to 250 (our maximum).",
      "factor": {
        "script": "construct_training_data",
        "parameter": "nwords",
        "labels": [
          "LLT Term Only", "6", "15", "30", "60", "125", "250"
        ]
      },
      "construct_training_data": {
        "nwords": [3, 6, 15, 30, 60, 125, 250]
      }
    },

    "2A": {
      "name": "Compare values for nwords, ref method 8 (87.5% after)",
      "description": "Same as experiment 2 except for reference method 8, rather than 0. The number of words we extract from the labels and use to make the examples is a core component of the reference set. Essentially the thinking is the more words the better, however, that also exponentially increases the running time. In this experiment we adjust the number of words used in the example strings from 30 up to 250 (our maximum).",
      "factor": {
        "script": "construct_training_data",
        "parameter": "nwords",
        "labels": [
          "6", "15", "30", "60", "125", "250"
        ]
      },
      "construct_training_data": {
        "method": [8],
        "nwords": [6, 15, 30, 60, 125, 250]
      },
      "fit_clinicalbert": {
        "epochs": [10]
      }
    },

    "2B": {
      "name": "Compare values for nwords, for ref method 14 (87.5% after, prepend ref source)",
      "description": "Same as experiment 2 except for reference method 8, rather than 0. The number of words we extract from the labels and use to make the examples is a core component of the reference set. Essentially the thinking is the more words the better, however, that also exponentially increases the running time. In this experiment we adjust the number of words used in the example strings from 30 up to 250 (our maximum).",
      "factor": {
        "script": "construct_training_data",
        "parameter": "nwords",
        "labels": [
          "6", "15", "30", "60", "125", "250"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [6, 15, 30, 60, 125, 250]
      },
      "fit_clinicalbert": {
        "epochs": [10]
      }
    },

    "3": {
      "name": "Compare grouping functions",
      "description": "Each drug, event pair has multiple example strings and each of these example strings will have its own prediction. In order to reduce this down to each drug, event pair we have to decide how to group the data together.",
      "factor": {
        "script": "compile_results",
        "parameter": "group-function",
        "labels": ["mean", "max", "median", "min"]
      },
      "compile_results": {
        "group-function": ["mean", "max", "median", "min"]
      }
    },

    "4": {
      "name": "Compare performance across AR, BW, and WP sections",
      "description": "Compare performance for each of the sections: Adverse Reactions (AR), Boxed Warnings (BW), and Warnings and Precautions (WP). For this experiment we train all of them independently from the pre-trained ClinicalBERT network. Also compare grouping all of the sections together. We may see improved performance of the lower frequency sections in this scenario.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "section",
        "labels": ["AR", "BW", "WP", "ALL", "ARBW"]
      },
      "construct_training_data": {
        "section": ["AR", "BW", "WP", "ALL", "ARBW"]
      }
    },

    "4A": {
      "name": "Compare performance across AR, BW, and WP sections, method 14 (prepend refsource, 87.5 After)",
      "description": "Compare performance for each of the sections: Adverse Reactions (AR), Boxed Warnings (BW), and Warnings and Precautions (WP). For this experiment we train all of them independently from the pre-trained ClinicalBERT network. Also compare grouping all of the sections together. We may see improved performance of the lower frequency sections in this scenario.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "section",
        "labels": ["AR", "BW", "WP", "ALL", "ARBW"]
      },
      "construct_training_data": {
        "section": ["AR", "BW", "WP", "ALL", "ARBW"],
        "method": [14]
      }
    },

    "4B": {
      "name": "Compare performance across AR, BW, and WP sections, PMB, method 14 (prepend refsource, 87.5 After)",
      "description": "Compare performance for each of the sections: Adverse Reactions (AR), Boxed Warnings (BW), and Warnings and Precautions (WP). For this experiment we train all of them independently from the pre-trained PubMedBERT network. Also compare grouping all of the sections together. We may see improved performance of the lower frequency sections in this scenario.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "section",
        "labels": ["AR", "BW", "WP", "ALL", "ARBW"]
      },
      "construct_training_data": {
        "section": ["AR", "BW", "WP", "ALL", "ARBW"],
        "method": [14]
      },
      "fit_clinicalbert": {
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "5": {
      "name": "Investigate pretraining with AR for Boxed Warning performance",
      "description": "Boxed warnings do not have as many training examples and suffer in performance. We want to investigate using the pretrained model from the Adverse Reactions sections with no additional training and with additional training to see how it performs.",
      "dependencies": ["4"],
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["network", "epochs"],
        "labels": [
          "PreTrain:AR, Fit 0 Epochs",
          "PreTrain:BW, Fit 0 Epochs",
          "PreTrain:ALL, Fit 0 Epochs",
          "PreTrain:AR, Fit 10 Epochs",
          "PreTrain:BW, Fit 10 Epochs",
          "PreTrain:ALL, Fit 10 Epochs"
        ]
      },
      "construct_training_data": {
        "section": ["BW"]
      },
      "fit_clinicalbert": {
        "network": [
          "bestepoch-bydrug-CB_0-AR-125-all_222_24_25_1e-06_256_32.pth",
          "bestepoch-bydrug-CB_0-BW-125-all_222_24_25_1e-06_256_32.pth",
          "bestepoch-bydrug-CB_0-ALL-125-all_222_24_25_1e-06_256_32.pth"
        ],
        "epochs": [0, 10]
      }
    },

    "5A": {
      "name": "Investigate pretraining with AR for Boxed Warning performance, PMB",
      "description": "Boxed warnings do not have as many training examples and suffer in performance. We want to investigate using the pretrained model from the Adverse Reactions sections with no additional training and with additional training to see how it performs. Same as Experiment 5 except using PubMedBERT as the pre-trained model.",
      "dependencies": ["4B"],
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["network", "epochs"],
        "labels": [
          "PreTrain:AR, Fit 0 Epochs",
          "PreTrain:BW, Fit 0 Epochs",
          "PreTrain:ALL, Fit 0 Epochs",
          "PreTrain:AR, Fit 10 Epochs",
          "PreTrain:BW, Fit 10 Epochs",
          "PreTrain:ALL, Fit 10 Epochs"
        ]
      },
      "construct_training_data": {
        "section": ["BW"],
        "method": [14]
      },
      "fit_clinicalbert": {
        "network": [
          "bestepoch-bydrug-PMB_14-AR-125-all_222_24_25_1e-06_256_32.pth",
          "bestepoch-bydrug-PMB_14-BW-125-all_222_24_25_1e-06_256_32.pth",
          "bestepoch-bydrug-PMB_14-ALL-125-all_222_24_25_1e-06_256_32.pth"
        ],
        "epochs": [0, 10]
      }
    },

    "6": {
      "name": "Compare string before to string after",
      "description": "REPLACED BY 6A. When constructing the training data we have the choice to take words preceding the event term, following the event term, or an event split of both (default). This experiment compares the performance of those three options. This could be part of experiment 0, but that seems to be getting crowded.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": [
          "After Only",
          "87.5% After",
          "75% After",
          "Before and After",
          "75% Before",
          "87.5% Before",
          "Before Only"
        ]
      },
      "construct_training_data": {
        "method": [7, 8, 9, 0, 10, 11, 6],
        "nwords": [30],
        "section": ["AR"]
      },
      "fit_clinicalbert": {
        "max-length": [32]
      }
    },

    "6A": {
      "name": "Compare string before to string after",
      "description": "Same as experiment 6, but with no max_length set. When constructing the training data we have the choice to take words preceding the event term, following the event term, or an event split of both (default). This experiment compares the performance of those three options. This could be part of experiment 0, but that seems to be getting crowded.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": [
          "After Only",
          "87.5% After",
          "75% After",
          "Before and After",
          "75% Before",
          "87.5% Before",
          "Before Only"
        ]
      },
      "construct_training_data": {
        "method": [7, 8, 9, 0, 10, 11, 6],
        "nwords": [30],
        "section": ["AR"]
      }
    },

    "6B": {
      "name": "Compare string before to string after with prepend refsource",
      "description": "Same as Experiment 6A but now including refsource which was found in Experiment 0 to add value.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": [
          "After Only",
          "87.5% After",
          "75% After",
          "Before and After"
        ]
      },
      "construct_training_data": {
        "method": [15, 14, 13, 12],
        "section": ["AR"]
      }
    },

    "7": {
      "name": "Explore learning rates, ClinicalBERT.",
      "description": "Evalaute several different learning rates for their impact on overall performance.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "learning-rate",
        "labels": [
          "1e-4",
          "1e-5",
          "1e-6",
          "1e-7",
          "1e-8"
        ]
      },
      "construct_training_data": {
        "method": [14]
      },
      "fit_clinicalbert": {
        "learning-rate": [1e-4, 1e-5, 1e-6, 1e-7, 1e-8]
      }
    },

    "7A": {
      "name": "Explore learning rates, PubMedBERT",
      "description": "Evalaute several different learning rates for their impact on overall performance.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "learning-rate",
        "labels": [
          "1e-4",
          "7.5e-5",
          "5e-5",
          "2.5e-5",
          "1e-5",
          "1e-6",
          "1e-7",
          "1e-8"
        ]
      },
      "construct_training_data": {
        "method": [14]
      },
      "fit_clinicalbert": {
        "learning-rate": [1e-4, 7.5e-5, 5e-5, 2.5e-5, 1e-5, 1e-6, 1e-7, 1e-8],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "7B": {
      "name": "Explore learning rates, PubMedBERT, nwords=60",
      "description": "Evalaute several different learning rates for their impact on overall performance.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "learning-rate",
        "labels": [
          "1e-4",
          "7.5e-5",
          "5e-5",
          "2.5e-5",
          "1e-5",
          "1e-6"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [60]
      },
      "fit_clinicalbert": {
        "learning-rate": [1e-4, 7.5e-5, 5e-5, 2.5e-5, 1e-5, 1e-6],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "8": {
      "name": "BW reference methods",
      "description": "Compare top three reference methods (0, 8, and 14) for Boxed Warnings. Note these are only trained on the Boxed Warnings examples. See 8A for a version where we evalaute performance of BW but trained on ALL (as was found in Experiment 4 to produce better results for the BW section).",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": ["0", "8", "14"]
      },
      "construct_training_data": {
        "method": [0, 8, 14],
        "section": ["BW"]
      }
    },

    "8A": {
      "name": "BW reference methods, trained on ALL",
      "description": "Compare top three reference methods (0, 8, and 14) for Boxed Warnings. This using the models trained on data from ALL sections and then the rows corresponding to Boxed Warnings are extracted. NOTE for Notebook Implementation: The extraction of the rows for BW will have to be done inline within the evaluate notebook. See Experiment 4 notebook for how to do this.",
      "factor": {
        "script": "construct_training_data",
        "parameter": "method",
        "labels": ["0", "8", "14"]
      },
      "construct_training_data": {
        "method": [0, 8, 14],
        "section": ["ALL"]
      }
    },

    "9": {
      "name": "ADR Term Identification Method",
      "description": "Compare different methods of identifying ADR terms from text, exact string match vs deepcadrme.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["refsource", "network"],
        "labels": [
          "All: ClinicalBERT",
          "Exact Only: ClinicalBERT",
          "All: PubMedBERT",
          "Exact Only: PubMedBERT"
        ]
      },
      "construct_training_data": {
        "method": [8]
      },
      "fit_clinicalbert": {
        "refsource": ["all", "exact"],
        "network": ["Bio_ClinicalBERT", "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "9A": {
      "name": "ADR Term Identification Method, prepend refsource",
      "description": "Compare different methods of identifying ADR terms from text, exact string match vs deepcadrme.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["refsource", "network"],
        "labels": [
          "All: ClinicalBERT",
          "Exact Only: ClinicalBERT",
          "All: PubMedBERT",
          "Exact Only: PubMedBERT"
        ]
      },
      "construct_training_data": {
        "method": [14]
      },
      "fit_clinicalbert": {
        "refsource": ["all", "exact"],
        "network": ["Bio_ClinicalBERT", "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10": {
      "name": "TAC Training/Testing Split",
      "description": "Use the TAC training and testing splits instead of randomly splitting.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["split-method", "network"],
        "labels": [
          "Default (80/10/10): CB",
          "TAC (45/5/50): CB",
          "Default (80/10/10): PMB",
          "TAC (45/5/50): PMB"
        ]
      },
      "construct_training_data": {
        "method": [8],
        "nwords": [125]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "network": ["Bio_ClinicalBERT", "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10A": {
      "name": "TAC Training/Testing Split - prepend refsource",
      "description": "Use the TAC training and testing splits instead of randomly splitting.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": ["split-method", "network"],
        "labels": [
          "Default (80/10/10): CB",
          "TAC (45/5/50): CB",
          "Default (80/10/10): PMB",
          "TAC (45/5/50): PMB"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [125]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "network": ["Bio_ClinicalBERT", "microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10B": {
      "name": "TAC Training/Testing Split - prepend refsource, nwords explore",
      "description": "Use the TAC training and testing splits instead of randomly splitting.",
      "factor": {
        "script": ["construct_training_data", "fit_clinicalbert"],
        "parameter": [["nwords"], ["split-method"]],
        "labels": [
          "Nwords=60, Default Split (80/10/10)",
          "Nwords=60, TAC (45/5/50)",
          "Nwords=125, Default Split (80/10/10)",
          "Nwords=125, TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [60, 125]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"]
      }
    },

    "10C": {
      "name": "TAC Training/Testing Split - PubMedBERT, prepend refsource, nwords explore",
      "description": "Use the TAC training and testing splits instead of randomly splitting. Network used here is PubMedBERT which was found to have better performance in head-to-head comparions (See Experiments 0, 0A, 1, and 10A).",
      "factor": {
        "script": ["construct_training_data", "fit_clinicalbert"],
        "parameter": [["nwords"], ["split-method"]],
        "labels": [
          "Nwords=60, Default Split (80/10/10)",
          "Nwords=60, TAC (45/5/50)",
          "Nwords=125, Default Split (80/10/10)",
          "Nwords=125, TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [60, 125]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10D": {
      "name": "TAC Training/Testing Split - Boxed Warnings: PubMedBERT, refmeth 14, nwords 60, ALL sections",
      "description": "Use the TAC training and testing splits instead of randomly splitting. Network used here is PubMedBERT which was found to have better performance in head-to-head comparions (See Experiments 0, 0A, 1, and 10A). Using 60 as nwords, found to have the best performance (See Experiments 2-2B and 10C). Training on ALL sections so that we can evaluate BW method on TAC gold standard.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "split-method",
        "labels": [
          "Default Split (80/10/10)",
          "TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [60],
        "section": ["ALL"]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10E": {
      "name": "TAC Training/Testing Split - PubMedBERT, prepend refsource",
      "description": "Use the TAC training and testing splits instead of randomly splitting. Network used here is PubMedBERT which was found to have better performance in head-to-head comparions (See Experiments 0, 0A, 1, and 10A).",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "split-method",
        "labels": [
          "Default Split (80/10/10)",
          "TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [125]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "learning-rate": [2.5e-5],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },

    "10F": {
      "name": "TAC Training/Testing Split - Boxed Warnings: PubMedBERT, refmeth 14, nwords 125, pretrained with ALL sections",
      "description": "Use the TAC training and testing splits instead of randomly splitting. Network used here is PubMedBERT which was found to have better performance in head-to-head comparions (See Experiments 0, 0A, 1, and 10A). Using 60 as nwords, found to have the best performance (See Experiments 2-2B and 10C). Training on ALL sections so that we can evaluate BW method on TAC gold standard.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "split-method",
        "labels": [
          "Default Split (80/10/10)",
          "TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [125],
        "section": ["BW"]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "learning-rate": [2.5e-5],
        "network": ["bestepoch-bydrug-PMB_14-ALL-125-all_222_24_25_1e-06_256_32.pth"]
      }
    },

    "10G": {
      "name": "TAC Training/Testing Split - Warnings and Precautions: PubMedBERT, refmeth 14, nwords 125, pretrained with ALL sections",
      "description": "Use the TAC training and testing splits instead of randomly splitting. Network used here is PubMedBERT which was found to have better performance in head-to-head comparions (See Experiments 0, 0A, 1, and 10A). Using 60 as nwords, found to have the best performance (See Experiments 2-2B and 10C). Training on ALL sections so that we can evaluate WP method on TAC gold standard.",
      "factor": {
        "script": "fit_clinicalbert",
        "parameter": "split-method",
        "labels": [
          "Default Split (80/10/10)",
          "TAC (45/5/50)"
        ]
      },
      "construct_training_data": {
        "method": [14],
        "nwords": [125],
        "section": ["WP"]
      },
      "fit_clinicalbert": {
        "split-method": ["24", "TAC"],
        "learning-rate": [2.5e-5],
        "network": ["bestepoch-bydrug-PMB_14-ALL-125-all_222_24_25_1e-06_256_32.pth"]
      }
    }

  },

  "replicates": {
  },

  "deployments": {
    "v1.0.0-AR": {
      "name": "OnSIDES Version 1.0.0 - Adverse Reactions",
      "description": "First major release of the model to extract side effect terms from the Adverse Reactions section of the labels, created on April 29, 2022. These paramters were chosen through analysis of the results of Experiments 1 through 7. Max F1 (threshold = 2.397) was 0.87 on the validation set and 0.86 on the test set. AUROC was 0.88 on the validation set and 0.88 on the test set. AUPR was 0.91 on the validation set and 0.91 on the test set.",
      "threshold": 2.397,
      "model": "bestepoch",
      "construct_training_data": {
        "method": [8],
        "nwords": [125],
        "section": ["AR"]
      },
      "fit_clinicalbert": {
        "epochs": [10],
        "refsource": ["exact"]
      }
    },
    "v1.0.0-BW": {
      "name": "OnSIDES Version 1.0.0 - Boxed Warnings",
      "description": "First major release of the model to extract side effect terms from the Boxed Warnings section of the labels, created on April 29, 2022. These paramters were chosen through analysis of the results of Experiments 1 through 7. Max F1 was 0.70 on the validation set and 0.66 on the test set. AUROC was 0.74 on the validation set and 0.71 on the test set. AUPR was 0.72 on the validation set and 0.60 on the test set.",
      "threshold": 2.794,
      "model": "bestepoch",
      "construct_training_data": {
        "method": [8],
        "nwords": [125],
        "section": ["BW"]
      },
      "fit_clinicalbert": {
        "epochs": [25],
        "refsource": ["exact"]
      }
    },
    "v2.1.0-AR": {
      "name": "OnSIDES Version 2.1.0 - Adverse Reactions",
      "description": "Second major release of the model to extract side effect terms from the Adverse Reactions section of the labels, created on November 10, 2022. These paramters were chosen through analysis of the results of Experiments 1 through 10. Max F1 (threshold = 0.4633) was 0.913 on the validation set and 0.898 on the test set. AUROC was 0.934 on the validation set and 0.922 on the test set. AUPR was 0.960 on the validation set and 0.948 on the test set. When evaluated against the TAC gold standard, the model achieves: F1=0.875, AUROC=0.912, and AUPR=0.937. Using the official eval script from TAC, the Macro-F1 is 85.40 and the Micro-F1 is 87.05. See notebook for Experiment 10E. Same as 2.0.0 version, updated to 2.1.0 to sync with new WP section.",
      "threshold": 0.4633,
      "model": "bestepoch",
      "construct_training_data": {
        "method": [14],
        "nwords": [125],
        "section": ["AR"]
      },
      "fit_clinicalbert": {
        "epochs": [25],
        "learning-rate": [2.5e-5],
        "network": ["microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract"]
      }
    },
    "v2.1.0-BW": {
      "name": "OnSIDES Version 2.1.0 - Boxed Warnings",
      "description": "Second major release of the model to extract side effect terms from the Boxed Warnings section of the labels, created on November 11, 2022. These paramters were chosen through analysis of the results of Experiments 1 through 10. Max F1 (threshold = 3.324) was 0.800 on the validation set and 0.714 on the test set. AUROC was 0.867 on the validation set and 0.849 on the test set. AUPR was 0.760 on the validation set and 0.722 on the test set. When evaluated against the TAC gold standard, the model achieves: F1=0.680, AUROC=0.808, AUPR=0.636. See notebook for Experiment 10F. Same as 2.0.0 version, updated to 2.1.0 to sync with new WP section.",
      "threshold": 3.324,
      "model": "bestepoch",
      "dependencies": ["4B"],
      "construct_training_data": {
        "method": [14],
        "nwords": [125],
        "section": ["BW"]
      },
      "fit_clinicalbert": {
        "epochs": [25],
        "learning-rate": [2.5e-5],
        "network": ["bestepoch-bydrug-PMB_14-ALL-125-all_222_24_25_1e-06_256_32.pth"]
      }
    },
    "v2.1.0-WP": {
      "name": "OnSIDES Version 2.1.0 - Warnings and Precautions",
      "description": "Second major release of the model to extract side effect terms from the Warnings and Precautions section of the labels, created on September 18, 2024. These paramters were chosen through analysis of the results of Experiments 1 through 10. Max F1 (threshold = 3.324) was 0.800 on the validation set and 0.714 on the test set. AUROC was 0.867 on the validation set and 0.849 on the test set. AUPR was 0.760 on the validation set and 0.722 on the test set. When evaluated against the TAC gold standard, the model achieves: F1=0.680, AUROC=0.808, AUPR=0.636. See notebook for Experiment 10G.",
      "threshold": 0.7328,
      "model": "final",
      "construct_training_data": {
        "method": [14],
        "nwords": [125],
        "section": ["WP"]
      },
      "fit_clinicalbert": {
        "epochs": [25],
        "learning-rate": [2.5e-5],
        "network": ["bestepoch-bydrug-PMB_14-ALL-125-all_222_24_25_1e-06_256_32.pth"]
      }
    }

  }
}