<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JFR</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Form Res</journal-id>
      <journal-title>JMIR Formative Research</journal-title>
      <issn pub-type="epub">2561-326X</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v8i1e58176</article-id>
      <article-id pub-id-type="pmid">39515814</article-id>
      <article-id pub-id-type="doi">10.2196/58176</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Crowdsourcing Adverse Events Associated With Monoclonal Antibodies Targeting Calcitonin Gene–Related Peptide Signaling for Migraine Prevention: Natural Language Processing Analysis of Social Media</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Mavragani</surname>
            <given-names>Amaryllis</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Wenzel</surname>
            <given-names>Richard</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Zhang</surname>
            <given-names>Pengfei</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Department of Neurology</institution>
            <institution>Rutgers-Robert Wood Johnson Medical School</institution>
            <addr-line>125 Paterson Street</addr-line>
            <addr-line>Suite 6200</addr-line>
            <addr-line>New Brunswick, NJ, 08901</addr-line>
            <country>United States</country>
            <phone>1 7322357729</phone>
            <email>pz124@rwjms.rutgers.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1132-1937</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Kamitaki</surname>
            <given-names>Brad K</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7600-7805</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Do</surname>
            <given-names>Thien Phu</given-names>
          </name>
          <degrees>MD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <xref rid="aff3" ref-type="aff">3</xref>
          <xref rid="aff4" ref-type="aff">4</xref>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9631-0665</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Neurology</institution>
        <institution>Rutgers-Robert Wood Johnson Medical School</institution>
        <addr-line>New Brunswick, NJ</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Neurology</institution>
        <institution>Danish Headache Center</institution>
        <institution>Copenhagen University Hospital - Rigshospitalet</institution>
        <addr-line>Copenhagen</addr-line>
        <country>Denmark</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Neurology</institution>
        <institution>Copenhagen University Hospital - Herlev and Gentofte</institution>
        <addr-line>Herlev</addr-line>
        <country>Denmark</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Danish Knowledge Center on Headache Disorders</institution>
        <addr-line>Glostrup</addr-line>
        <country>Denmark</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>Department of Clinical Medicine</institution>
        <institution>University of Copenhagen</institution>
        <addr-line>Copenhagen</addr-line>
        <country>Denmark</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Pengfei Zhang <email>pz124@rwjms.rutgers.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2024</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>8</day>
        <month>11</month>
        <year>2024</year>
      </pub-date>
      <volume>8</volume>
      <elocation-id>e58176</elocation-id>
      <history>
        <date date-type="received">
          <day>8</day>
          <month>3</month>
          <year>2024</year>
        </date>
        <date date-type="rev-request">
          <day>6</day>
          <month>6</month>
          <year>2024</year>
        </date>
        <date date-type="rev-recd">
          <day>23</day>
          <month>6</month>
          <year>2024</year>
        </date>
        <date date-type="accepted">
          <day>10</day>
          <month>7</month>
          <year>2024</year>
        </date>
      </history>
      <copyright-statement>©Pengfei Zhang, Brad K Kamitaki, Thien Phu Do. Originally published in JMIR Formative Research (https://formative.jmir.org), 08.11.2024.</copyright-statement>
      <copyright-year>2024</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Formative Research, is properly cited. The complete bibliographic information, a link to the original publication on https://formative.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://formative.jmir.org/2024/1/e58176" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Clinical trials demonstrate the efficacy and tolerability of medications targeting calcitonin gene–related peptide (CGRP) signaling for migraine prevention. However, these trials may not accurately reflect the real-world experiences of more diverse and heterogeneous patient populations, who often have higher disease burden and more comorbidities. Therefore, postmarketing safety surveillance is warranted. Regulatory organizations encourage marketing authorization holders to screen digital media for suspected adverse reactions, applying the same requirements as for spontaneous reports. Real-world data from social media platforms constitute a potential venue to capture diverse patient experiences and help detect treatment-related adverse events. However, while social media holds promise for this purpose, its use in pharmacovigilance is still in its early stages. Computational linguistics, which involves the automatic manipulation and quantitative analysis of oral or written language, offers a potential method for exploring this content.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aims to characterize adverse events related to monoclonal antibodies targeting CGRP signaling on Reddit, a large online social media forum, by using computational linguistics.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>We examined differences in word frequencies from medication-related posts on the Reddit subforum r/Migraine over a 10-year period (2010-2020) using computational linguistics. The study had 2 phases: a validation phase and an application phase. In the validation phase, we compared posts about propranolol and topiramate, as well as posts about each medication against randomly selected posts, to identify known and expected adverse events. In the application phase, we analyzed posts discussing 2 monoclonal antibodies targeting CGRP signaling—erenumab and fremanezumab—to identify potential adverse events for these medications.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>From 22,467 Reddit r/Migraine posts, we extracted 402 (2%) propranolol posts, 1423 (6.33%) topiramate posts, 468 (2.08%) erenumab posts, and 73 (0.32%) fremanezumab posts. Comparing topiramate against propranolol identified several expected adverse events, for example, “appetite,” “weight,” “taste,” “foggy,” “forgetful,” and “dizziness.” Comparing erenumab against a random selection of terms identified “constipation” as a recurring keyword. Comparing erenumab against fremanezumab identified “constipation,” “depression,” “vomiting,” and “muscle” as keywords. No adverse events were identified for fremanezumab.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>The validation phase of our study accurately identified common adverse events for oral migraine preventive medications. For example, typical adverse events such as “appetite” and “dizziness” were mentioned in posts about topiramate. When we applied this methodology to monoclonal antibodies targeting CGRP or its receptor—fremanezumab and erenumab, respectively—we found no definite adverse events for fremanezumab. However, notable flagged words for erenumab included “constipation,” “depression,” and “vomiting.” In conclusion, computational linguistics applied to social media may help identify potential adverse events for novel therapeutics. While social media data show promise for pharmacovigilance, further work is needed to improve its reliability and usability.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>internet</kwd>
        <kwd>patient reported outcome</kwd>
        <kwd>headache</kwd>
        <kwd>health information</kwd>
        <kwd>Reddit</kwd>
        <kwd>registry</kwd>
        <kwd>monoclonal antibody</kwd>
        <kwd>crowdsourcing</kwd>
        <kwd>postmarketing</kwd>
        <kwd>safety</kwd>
        <kwd>surveillance</kwd>
        <kwd>migraine</kwd>
        <kwd>preventives</kwd>
        <kwd>prevention</kwd>
        <kwd>self-reported</kwd>
        <kwd>calcitonin gene–related peptide</kwd>
        <kwd>calcitonin</kwd>
        <kwd>therapeutics</kwd>
        <kwd>social media</kwd>
        <kwd>medication-related</kwd>
        <kwd>posts</kwd>
        <kwd>propranolol</kwd>
        <kwd>topiramate</kwd>
        <kwd>erenumab</kwd>
        <kwd>fremanezumab</kwd>
        <kwd>cross-sectional</kwd>
        <kwd>surveys</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Preventive medications for migraine include both non–disease-specific drugs (eg, antihypertensives and anticonvulsants) and, more recently, disease-specific anti–calcitonin gene–related peptide (CGRP) therapies [<xref ref-type="bibr" rid="ref1">1</xref>]. Randomized controlled trials (RCTs) demonstrate the clinically relevant efficacy and tolerability of anti-CGRP monoclonal antibodies (mAbs) [<xref ref-type="bibr" rid="ref2">2</xref>]. However, they may not accurately reflect the real-world experiences of more heterogeneous and diverse patient populations who often have a higher disease burden and more comorbidities [<xref ref-type="bibr" rid="ref3">3</xref>-<xref ref-type="bibr" rid="ref6">6</xref>]. In addition to direct reports of potential adverse events from health care providers and patients, regulatory organizations encourage marketing authorization holders to screen digital media for suspected adverse reactions, which should be handled similarly to spontaneous reports with the same requirements applied [<xref ref-type="bibr" rid="ref7">7</xref>]. While social media represents an obvious venue in this context, its usage for pharmacovigilance is still in its infancy [<xref ref-type="bibr" rid="ref8">8</xref>]. Complementary methods of postmarketing surveillance for adverse events with social media are needed.</p>
      <p>The last 15 years were characterized by a vast proliferation of social media platforms, allowing for both the rapid dissemination of medical information, as well as direct insight into patient perspectives. Indeed, migraine is most prevalent in individuals between 15-49 years of age, who are among the most active users of the internet [<xref ref-type="bibr" rid="ref9">9</xref>]. This is also reflected by the growth of migraine-related content on the internet [<xref ref-type="bibr" rid="ref10">10</xref>-<xref ref-type="bibr" rid="ref12">12</xref>]. The anonymous social media discussion board Reddit has over 430 million monthly active users and serves as a popular forum for health-related discussions [<xref ref-type="bibr" rid="ref13">13</xref>]. Unlike other digital media platforms, Reddit provides easily accessible and legal ways of accessing historic forum data. Anonymous forums like Reddit allow users to share personal medical experiences and advice without social stigma. Collectively, the migraine-related subforums (or “subreddits”) such as r/Migraine have a user base exceeding 54,100. While these platforms allow for a democratic exploration of patient experiences and perspectives, they are relatively understudied.</p>
      <p>Computational linguistics, a discipline encompassing automatic manipulation and quantitative analysis of oral or written language, represents a potential way to explore this content. Computational linguistics can be applied to identify clusters of words in a dataset with the goal of deriving dominant thematic elements. Identifying word distribution frequencies in specific groups of texts is a common technique used in computational linguistics with multiple wide-ranging applications [<xref ref-type="bibr" rid="ref14">14</xref>]. In this study, we applied computational linguistics, specifically word frequency analysis, to characterize and identify self-reported adverse events associated with the new therapeutic anti-CGRP mAbs reported by users on Reddit. To ascertain the reliability and validity of our methodology, we first conducted a preliminary study with older-generation migraine drugs to see if we could identify known adverse reactions. After validation, we applied the methodology to the newer generation of CGRP therapies for migraine prevention.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Study Design</title>
        <p>This study was a cross-sectional survey of posts on the r/Migraine subforum on Reddit. Data were analyzed using custom back-end software. The study consisted of four phases: (1) data extraction, (2) automated data organization, (3) validation of computational linguistics approach, and (4) application of computational linguistics approach.</p>
        <sec>
          <title>Data Extraction</title>
          <p>We extracted posts in JavaScript Object Notation format from the Reddit r/Migraine subforum published from January 1, 2010, to January 1, 2020, using Pushshift, a publicly available Reddit repository, in order to extract data for analysis [<xref ref-type="bibr" rid="ref15">15</xref>]. The time stamp, title, and text of each post were obtained and combined into a comma-separated values file for further evaluation. Of note, our search did not include any comments responding to individual posts. Furthermore, usernames were excluded to ensure the protection of personal data. Once the above data were downloaded, we then converted the entire document to lowercase characters for ease of data search and extraction.</p>
        </sec>
        <sec>
          <title>Automated Data Organization</title>
          <p>For the validation phase of the study, we algorithmically obtained, through the Haskell filter function, a list of documents containing posts with either the brand or trade name for propranolol and topiramate. These medications were selected as evidence-based preventive treatments for migraine in accordance with American Headache Society guidelines and because their side effect profiles are well known [<xref ref-type="bibr" rid="ref16">16</xref>]. Next, we algorithmically compiled all posts containing each drug in separate documents. To use computational linguistics terminology, each document represents a corpus for a medication; each post within that corpus further represents a text.</p>
          <p>We subsequently identified posts not including any of the above medications, nor including any CGRP therapeutic medication, to serve as controls for comparison. To conduct multiple tests, we randomly divided these controls into 4 groups: group 1, group 2, group 3, and group 4. To accomplish this, we assigned each post in the control a random number between 1 to 4. All posts with randomly assigned digits 1 were grouped into group 1. Those with a digit of 2 were grouped into group 2, and so on.</p>
        </sec>
        <sec>
          <title>Validation of Computational Linguistics Approach</title>
          <p>Comparing the relative occurrence of words between 2 groups of texts is a well-defined technique in computational linguistics. For drugs with different distinct side effect profiles (eg, the anticonvulsant topiramate vs the beta-blocker propranolol), we hypothesized there would be a difference in the frequency of word distributions between corpora for these different medications.</p>
          <p>Although the chi-square test or log-likelihood ratio test are commonly used, Lijiffijit et al [<xref ref-type="bibr" rid="ref17">17</xref>] recently demonstrated that the Wilcoxon rank-sum test, Welch <italic>t</italic> test, or bootstrap test are better choices to use to avoid overconfidence bias. Consequently, we used and validated Welch <italic>t</italic> test for each of these comparisons given the ease of implementation and intuitiveness.</p>
          <p>In accordance with the abovementioned methodology, given a corpus for drug S and a corpus for drug T, we denoted each individual word by q, the total number of words in each corpus by n<sub>1</sub> and n<sub>2</sub>, and x<sub>1</sub> and x<sub>2</sub> as the mean frequency of q over corpus S and corpus T, respectively. We then applied Welch <italic>t</italic> test [<xref ref-type="bibr" rid="ref17">17</xref>]. To simplify our calculations, words with fewer than 3 characters in length, symbols, and numbers were filtered out. We identified any words showing statistical significance at <italic>P</italic>&#60;.05 using a 1-tailed <italic>t</italic> test since to reject the null hypothesis, it is sufficient to show that a specific word occurrence is greater—not less—in 1 corpus against another, or a <italic>t</italic> statistic&#62;1.73. Because this process involved the repeated application of Welch <italic>t</italic> test n<sub>1</sub>+ n<sub>2</sub> times to the same dataset, the Benjamini-Hochberg procedure was used to decrease multisampling bias. Specifically, once statistically significant words were obtained after a given comparison between 2 corpora, the list of significant words was then ranked by <italic>P</italic> value. We then applied the Benjamini-Hochberg procedure to this ranked ordered list, only retaining those words passing this procedure.</p>
          <p>To validate our method and the reliability of the Reddit data, we applied this approach first by comparing propranolol and topiramate. Of note, if a text existed in both the propranolol and the topiramate corpora, then it was taken out of both. Subsequently, we compared propranolol versus group 1 and topiramate versus group 2.</p>
        </sec>
        <sec>
          <title>Application of Computational Linguistics Approach</title>
          <p>The above approach was then applied to the corpus between erenumab versus group 3 in order to establish a baseline side effect profile for erenumab. Similarly, fremanezumab was compared to group 4 to establish a baseline side effect profile for fremanezumab. We then compared the erenumab corpus against the fremanezumab corpus to ascertain any difference between the 2 medication side effect profiles. This is clinically relevant, as erenumab targets the CGRP receptor, while fremanezumab targets the CGRP ligand. We were interested to see whether there were fewer reports of specific adverse events with erenumab than with fremanezumab. Again, when comparing erenumab and fremanezumab, shared posts were removed.</p>
          <p>For evaluation of our data, all contributing authors reviewed words identified through the above processes, where a word is counted as describing a potential adverse event by consensus. The above algorithms, including data download, parsing, filtering, and data analysis were implemented through custom back-end software using a combination of the Clojure and Haskell programming languages.</p>
        </sec>
      </sec>
      <sec>
        <title>Ethical Considerations</title>
        <p>Our data source was obtained from Pushshift, a social media data collection site, with the explicit purpose of collecting Reddit data for the purpose of research [<xref ref-type="bibr" rid="ref15">15</xref>]. Our project proposal was approved by the Rutgers University institutional review board as nonhuman subjects research (Pro2020001648). To ensure the anonymity of users in our data processing, we removed all usernames from our dataset. Furthermore, since our methodology examines posts in aggregate, individual identifiable data were not published nor evaluated as part of this study.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Dataset</title>
        <p>We extracted a total of 22,467 Reddit posts from r/Migraine, spanning from January 1, 2010, to December 31, 2019. In the validation phase of the study, we identified 402 (2%) posts that mentioned propranolol and 1423 (6.33%) posts that mentioned topiramate. During the application phase, focusing on mAbs targeting CGRP signaling, we found 468 (2.08%) posts that mentioned erenumab and 73 (0.32%) posts that mentioned fremanezumab (<xref rid="figure1" ref-type="fig">Figure 1</xref>).</p>
        <p>Since these posts did not exclusively reference the target medications, additional steps were taken to exclude co-occurring texts when comparing medications, as detailed in the <italic>Methods</italic> section. For the control corpora, group 1 contained 5058 posts, group 2 had 4996 posts, group 3 included 5130 posts, and group 4 comprised 5009 posts.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Study flow diagram of the validation and application phases of a computer linguistics analysis of r/Migraine social media posts on Reddit.</p>
          </caption>
          <graphic xlink:href="formative_v8i1e58176_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Validation of Computational Linguistics Approach</title>
        <p>When topiramate posts were compared against propranolol posts using the earlier methodology, side effect–related words including “appetite,” “cognitive,” “tingling,” “remember,” “taste,” “dopamax” (a frequently used slang term describing the cognitive adverse events of topiramate), “tastes,” and “kidney” were statistically more frequent (<xref ref-type="boxed-text" rid="box1">Textbox 1</xref>).</p>
        <boxed-text id="box1" position="float">
          <title>Keywords identified when comparing topiramate with propranolol from the Reddit r/Migraine subforum (2010-2020). Italicized words were identified as potential side effects–related words of interest.</title>
          <p>topamax, topiramate, <italic>appetite</italic>, effects, posts, ready, realized, serious, throw, <italic>cognitive</italic>, loss, side, treating, level, flat, <italic>tingling</italic>, situation, blocks, opinion, improved, ntopamax, options, woman, possibility, issues, reddit, glasses, awesome, cymbalta, grateful, <italic>remember</italic>, version, depressants, decent, dumb, believe, future, <italic>taste</italic>, angry, evidence, <italic>dopamax</italic>, tapering, someone, practice, trust, shortly, honest, sign, <italic>kidney</italic>, steroids, refused, tons, disorders, cambia, learned, nerve, feet, ahead, essentially, posted, <italic>tastes</italic>, allergic, emotional, harder, ride, limited, white, spoke, parts, comment, optimistic, hurt, count</p>
        </boxed-text>
        <p>When propranolol was compared to the control corpus group 1, a total of 98 words appeared more frequently, none of which we identified as potential adverse events (<xref ref-type="boxed-text" rid="box2">Textbox 2</xref>). When topiramate was compared against group 2, there were 563 words that appeared more often (<xref ref-type="boxed-text" rid="box3">Textbox 3</xref>). The following words were determined to be potential adverse effects: “appetite,” “weight,” “remember,” “memory,” “cognitive,” “anxiety,” “taste,” “zombie,” “tingly,” “dizzy,” “fatigue,” “forget,” “kidney,” “foggy,” “dopamax,” “forgetful,” “dizziness,” “carbonate,” and “tastes.”</p>
        <boxed-text id="box2" position="float">
          <title>Keywords identified when comparing propranolol with group 1 from the Reddit r/Migraine subforum (2010-2020).</title>
          <p>propranolol, taking, been, months, topamax, effects, prescribed, tried, with, take, medication, doctor, side, dose, have, neurologist, preventative, amitriptyline, daily, since, also, inderal, currently, anyone, started, month, years, weeks, from, beta, which, migraines, medications, some, about, life, week, meds, made, drug, really, time, year, days, will, other, first, nortriptyline, after, because, dosage, effect, again, just, didn, stopped, they, haven, topiramate, help, well, terrible, went, starting, this, past, blockers, still, doesn, almost, maxalt, nothing, feel, sumatriptan, that, back, like, seemed, what, stop, preventatives, appointment, long, good, advice, weight, blood, npropranolol, give, neuro, blocker, relief, imitrex, worked, results, going, headaches, only</p>
        </boxed-text>
        <boxed-text id="box3" position="float">
          <title>Keywords identified when comparing topiramate with group 2 from the Reddit r/Migraine subforum (2010-2020). Italicized words were identified as potential side effect–related words of interest.</title>
          <p>topamax, side, effects, taking, topiramate, been, have, with, neurologist, just, dose, about, that, take, started, tried, week, this, since, which, months, doctor, because, month, migraines, weeks, from, also, some, well, daily, other, prescribed, like, years, going, back, what, effect, really, preventative, only, work, working, neuro, days, made, know, much, long, loss, starting, need, even, after, currently, first, medication, meds, still, worked, didn, time, then, propranolol, again, good, stopped, taken, more, nothing, point, <italic>appetite</italic>, dosage, drug, imitrex, would, helped, when, control, works, medications, getting, night, will, think, preventatives, already, were, there, every, little, <italic>weight</italic>, went, amitriptyline, gave, said, issues, appointment, maxalt, increased, want, last, should, feel, anything, anyone, trying, seems, being, before, pretty, very, having, wants, here, finally, twice, until, able, give, helping, <italic>remember</italic>, while, chronic, where, experiences, haven, experience, next, over, three, year, triptans, stop, used, options, down, botox, could, severe, into, better, keep, they, beta, away, help, read, worse, blockers, birth, through, great, them, feet, <italic>memory</italic>, success, depakote, though, another, almost, increase, past, doses, under, female, anymore, these, frequency, decided, seeing, same, many, effective, switch, came, symptoms, actually, gabapentin, magnesium, different, without, needed, aimovig, else, seem, depression, lost, course, things, recently, thing, half, trokendi, lose, advice, desperate, wanted, problems, increasing, horrible, switched, diagnosed, couldn, took, normal, life, <italic>tingling</italic>, history, start, suggested, mood, drugs, several, sure, doesn, maybe, concerned, awful, <italic>cognitive</italic>, constant, might, find, <italic>anxiety</italic>, insurance, wasn, changed, called, experiencing, break, everything, feeling, doing, agreed, medicine, reading, function, never, makes, drive, neurologists, abortive, nthanks, talk, times, sent, headaches, ntopamax, myself, anti, nortriptyline, pill, treatment, verapamil, prescription, instead, results, than, told, once, withdrawal, prevention, nervous, supposed, august, doctors, seen, relief, probably, including, less, completely, worth, hands, felt, switching, nausea, finding, primary, care, reaction, pills, either, hoping, tired, change, least, antidepressants, continue, prescribe, april, something, plus, fioricet, background, post, period, current, basically, those, hell, deal, frustrated, hello, difference, question, sleepy, afraid, however, story, <italic>taste</italic>, around, cannot, list, terrible, says, cymbalta, wondering, stopping, suggestions, idea, medical, asked, <italic>zombie</italic>, fine, issue, such, serious, hard, combo, realize, right, changes, july, blood, body, pharmacist, initially, brain, ready, thinks, treating, happy, tonight, gone, suffering, people, done, developed, talked, <italic>tingly</italic>, abortives, supplements, couple, risk, small, frequent, stories, flat, became, eating, looking, zomig, stuff, free, gained, problem, stayed, <italic>dizzy</italic>, <italic>fatigue</italic>, opinion, wean, except, given, hair, gain, situation, possible, lasted, various, pounds, zero, words, causing, minimal, metoprolol, <italic>forget</italic>, reduced, taper, steroids, physician, relpax, definitely, single, upped, <italic>kidney</italic>, ntopiramate, possibly, eventually, weaning, steroid, elavil, tests, making, nafter, whole, exercise, insomnia, january, ever, zonegran, september, pain, thinking, second, generic, slowly, depressants, follow, major, <italic>foggy</italic>, <italic>dopamax</italic>, please, specialist, nsaids, level, functional, plan, monday, hospital, shot, drink, kept, reduction, toradol, none, surprised, gotten, cycle, rescue, positive, seemed, intractable, stay, make, failed, chance, dropped, ncurrently, nightly, crazy, live, acupuncture, average, family, seizure, <italic>forgetful</italic>, levels, quit, stomach, calcium, ability, early, practice, selftext, others, thanks, discuss, occasionally, june, propanolol, high, most, decrease, schedule, losing, gradually, believe, guess, both, stones, coming, bring, march, everyone, look, biggest, previous, option, higher, helps, found, longer, tolerance, october, onto, slow, morning, previously, constantly, wits, fairly, negative, especially, tapering, proven, posts, quitting, upping, common, kind, reason, possibility, number, waiting, online, bedtime, goes, name, pharmacy, best, super, improved, handle, treat, nbut, <italic>dizziness</italic>, using, thank, complete, main, vitamins, fast, stick, anyway, bumped, willing, miracle, hasn, shocked, clinic, decent, mris, noticed, weaned, refused, talking, candidate, together, quick, nanyway, occasional, round, efficacy, tolerate, improvement, <italic>carbonated</italic>, nany, suggest, chest, <italic>tastes</italic>, psychiatrist</p>
        </boxed-text>
      </sec>
      <sec>
        <title>Application of Computational Linguistics Approach</title>
        <p>We next identified frequent words of interest by comparing erenumab to group 3 (<xref ref-type="boxed-text" rid="box4">Textbox 4</xref>) and fremanezumab to group 4. For erenumab, 36 words appeared more frequently, but “constipation” was the only potential adverse effect. The only word appearing more often for fremanezumab versus the control corpus was “Ajovy,” a brand name for fremanezumab.</p>
        <boxed-text id="box4" position="float">
          <title>Keywords identified when comparing erenumab with group 3 from the Reddit r/Migraine subforum (2010-2020). Italicized words were identified as potential side effects–related words of interest.</title>
          <p>aimovig, dose, month, injection, botox, neurologist, effects, months, drug, first, insurance, emgality, injections, tried, topamax, side, free, trial, neuro, next, shot, approved, preventatives, since, hoping, ajovy, cgrp, <italic>constipation</italic>, effect, plan, working, here, received, shots, doses, propranolol</p>
        </boxed-text>
        <p>When comparing erenumab to fremanezumab, there were 180 words that were significantly associated with erenumab, with the following words representing potential adverse events: “depression,” “constipation,” and “vomiting” (<xref ref-type="boxed-text" rid="box5">Textbox 5</xref>). Notably, the word “muscle” was also associated with erenumab, but it was unclear whether this described an adverse event (<xref ref-type="boxed-text" rid="box5">Textbox 5</xref>). The only word considered significant for fremanezumab was “Ajovy” with no adverse effect–related words detected.</p>
        <boxed-text id="box5" position="float">
          <title>Keywords identified when comparing erenumab with fremanezumab from the Reddit r/Migraine subforum (2010-2020). Italicized words were identified as potential side effect–related words of interest.</title>
          <p>aimovig, where, felt, medications, doesn, effect, much, starting, makes, enough, stopped, such, come, everything, home, triptan, next, possible, half, wait, worth, being, live, high, talk, preventatives, trial, question, nhas, medicine, diagnosed, thinking, hour, means, received, know, definitely, best, caused, into, happy, sumatriptan, given, lost, issue, short, here, <italic>depression</italic>, frustrated, early, should, weird, aren, knows, problem, especially, leave, advance, mind, ones, painful, story, <italic>constipation</italic>, real, cost, school, list, thank, related, note, support, told, nothing, this, follow, nbut, nthis, stories, wouldn, another, available, https, thread, difficult, info, okay, regular, bring, vision, wrong, migraine, specialist, often, made, abortive, considering, history, intense, notice, sometimes, sort, <italic>vomiting</italic>, break, causing, expensive, experiencing, learned, middle, asking, physical, seemed, vent, muscle, coming, concerned, monday, water, current, program, work, changes, medical, make, didn, blocks, happen, recommended, along, case, horrible, extra, specific, chest, fact, fall, letting, nalso, reason, types, type, serious, reading, barely, situation, therapy, treatments, erenumab, nand, began, clinic, dealing, minute, trials, freaking, ideas, tuesday, incredibly, move, nfor, paying, strong, benefits, continue, friends, mostly, naimovig, ndoes, news, nnow, rest, unfortunately, wonder, hold, reddit, update, toradol, prescribe, disease, switched, rebound</p>
        </boxed-text>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>In this study, we applied computational linguistics methods to unfiltered discussions about migraine treatments on Reddit to distinguish patient-reported adverse effects associated with these medications. The validation phase of our study accurately identified common adverse events for standard migraine preventive treatments. For example, typical adverse events such as “taste” and “tingling” appeared in posts for topiramate. After applying this methodology to mAbs targeting CGRP or its receptor—fremanezumab and erenumab, respectively—no definite adverse events were flagged for fremanezumab. Notable flagged words for erenumab included “constipation,” “depression,” and “vomiting.”</p>
      <p>In our proof-of-concept analysis, we confirmed that keywords discussed in relation to topiramate from the Reddit r/Migraine subforum are consistent with the most common adverse events observed in RCTs [<xref ref-type="bibr" rid="ref18">18</xref>]. Interestingly, dizziness, a common side effect, did not appear when comparing propranolol to either the control corpus or topiramate [<xref ref-type="bibr" rid="ref19">19</xref>]. A possible explanation could be that many different words were used to describe this side effect, for example, “lightheadedness” and “dizzy,” which were detected individually but did not meet statistical significance after correcting for multiple comparisons. Because patients may use a variety of different words to describe a single symptom, pooling these terms be useful in future studies when analyzing social media data.</p>
      <p>While constipation was not a major concern during RCTs with erenumab, emerging real-world data report that it is among the most frequent reasons for treatment discontinuation [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>]. Constipation was subsequently added to the US Food and Drug Administration (FDA) label after regulatory approval [<xref ref-type="bibr" rid="ref22">22</xref>]. During RCTs, approximately 2% of participants reported constipation compared to approximately 10% to 20% of real-world users [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. Our findings further support this, since Reddit users frequently discuss constipation and vomiting in relation to erenumab. The discrepancy between data from RCTs and real-world experiences may be due to a more heterogeneous patient population receiving erenumab in the postmarketing phase. This population likely includes patients with other medical comorbidities or factors that could have excluded them from consideration in clinical trials. Increasing evidence suggests that individuals with migraine may have a higher prevalence of comorbid gastrointestinal disorders when compared with the general population [<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref25">25</xref>].</p>
      <p>Interestingly, the word “muscle” was associated with posts discussing erenumab when compared with fremanezumab. Although we did not include this word as a potential adverse event due to its ambiguity, it could be related to muscle cramps, which were added to the FDA warning label for erenumab after approval [<xref ref-type="bibr" rid="ref22">22</xref>]. Muscle spasms occurred in approximately 2% of patients in RCTs [<xref ref-type="bibr" rid="ref20">20</xref>]. Whether “muscle”-related adverse effects are occurring at a more significant rate in real-world use is unclear. Additional investigation into the extent and severity of muscle disease due to erenumab could be considered in future studies or when examining other postmarketing surveillance data sources for verification.</p>
      <p>Another notable postmarketing safety update was the addition of hypertension to the FDA label for erenumab; however, this was not flagged in our analysis. In RCTs, less than 1% of participants reported hypertension, but participants with severe cardiovascular risk factors were excluded from these trials, and hypertension has not been commonly noted in other studies [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. It is possible that cardiovascular risk factors are screened to a higher degree than gastrointestinal disorders. Patients at risk for hypertensive complications might, therefore, be more likely to decline treatment with this drug. Finally, worsening of hypertension is typically asymptomatic, and patients may not recognize this as a potential adverse event to raise in social media discussions.</p>
      <p>There were no keywords flagged in the fremanezumab dataset. The finding that constipation was flagged for erenumab but not for fremanezumab could be related to pharmacodynamic differences in the mechanism of action. Erenumab targets the CGRP receptor, while fremanezumab targets its ligand. One study offers a possible mechanistic model that strengthens this hypothesis, specifically, via direct effects of CGRP on intestinal motility modulated by other receptors in addition to the canonical CGRP receptor [<xref ref-type="bibr" rid="ref26">26</xref>]. However, galcanezumab, another mAb targeting the CGRP ligand, also had constipation listed as a common adverse reaction in clinical studies and postmarketing reports [<xref ref-type="bibr" rid="ref27">27</xref>]. Further data are needed to clarify any potential differences in mechanism.</p>
      <p>A strength of our study is the inherent anonymity of the Reddit forum, which allows us to analyze information that patients might not otherwise feel comfortable discussing or prioritize sharing during a clinical consultation. Querying the Reddit migraine subforum also provides insight into the experience of more than 54,000 unique users, which could include diverse sources ranging from patients and family members to experienced health care providers.</p>
      <p>However, the use of Reddit data also confers significant limitations. We cannot correlate our findings with individual patient demographics, previous medical history, or other clinically relevant information. All information is self-reported and subjective, which calls into question the accuracy of unverifiable data. Although the number of posts differs between erenumab and fremanezumab due to FDA approval time, we do not expect this to significantly change our result since word distribution should be theoretically constant despite the sample size. Another limitation is the use of multiple different words for the same symptom; for example, when “lightheadedness,” “dizzy,” and “dizziness” collectively describe the same entity. Relevant symptoms described in various ways may be missed during this type of analysis. Pooling multiple similar terms could avoid this problem in future studies. Finally, our method did not analyze comments responding to specific posts; incorporating this into future studies using word frequency methodology could increase the accuracy of our conclusions. Future directions for our line of research include the analysis of other digital platforms to compare with our findings.</p>
      <p>Postmarketing pharmacovigilance remains critical, especially when RCTs underdetect adverse events. When compared with health care professionals, direct patient reporting of adverse drug events identifies more potential signals and in greater detail [<xref ref-type="bibr" rid="ref28">28</xref>]. Our study provides support that monitoring patient reports via social media can be incorporated into postmarketing safety surveillance for preventive migraine medications. While our proposed methodology has its limitations, the data offer a broader postmarketing perspective that complements clinical trial findings and case reports. We hope that consideration by relevant stakeholders will enhance existing tools in the pharmacovigilance armamentarium, enabling quicker and earlier detection of adverse effect signals. However, more work is needed to improve the reliability and validity of social media analysis, which faces the same challenges inherent to big data, including high volume, unstructured capture, and structural regulatory barriers, which all need to be addressed before implementation [<xref ref-type="bibr" rid="ref29">29</xref>].</p>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">CGRP</term>
          <def>
            <p>calcitonin gene–related peptide</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">FDA</term>
          <def>
            <p>Food and Drug Administration</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">mAb</term>
          <def>
            <p>monoclonal antibody</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">RCT</term>
          <def>
            <p>randomized controlled trial</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>BKK has received research grant support from the American Epilepsy Society, the New Jersey Health Foundation, and the Resource Center for Alzheimer’s and Dementia Research in Asian and Pacific Americans under NIH/NIA Grant P30-AG083257. The contents of this paper are solely the responsibility of the authors and do not necessarily represent the official views of the National Institutes of Health (NIH).</p>
    </ack>
    <notes>
      <sec>
        <title>Data Availability</title>
        <p>The datasets generated during and/or analyzed during this study are available from the corresponding author on reasonable request.</p>
      </sec>
    </notes>
    <fn-group>
      <fn fn-type="conflict">
        <p>TPD declared no competing interests. BKK declared no competing interests. PZ has received honorariums from Lundbeck Biopharmaceuticals, Board Vitals, and Fieve Clinical Research. PZ collaborates with Headache Science Incorporated without receiving financial support. PZ has ownership interest in Cymbeline LLC.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Buse</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Pozo-Rosich</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Peres</surname>
              <given-names>MFP</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Terwindt</surname>
              <given-names>GM</given-names>
            </name>
            <name name-style="western">
              <surname>Halker Singh</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Tassorelli</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Do</surname>
              <given-names>TP</given-names>
            </name>
            <name name-style="western">
              <surname>Mitsikostas</surname>
              <given-names>DD</given-names>
            </name>
            <name name-style="western">
              <surname>Dodick</surname>
              <given-names>DW</given-names>
            </name>
          </person-group>
          <article-title>Migraine: integrated approaches to clinical management and emerging treatments</article-title>
          <source>Lancet</source>
          <year>2021</year>
          <volume>397</volume>
          <issue>10283</issue>
          <fpage>1505</fpage>
          <lpage>1518</lpage>
          <pub-id pub-id-type="doi">10.1016/S0140-6736(20)32342-4</pub-id>
          <pub-id pub-id-type="medline">33773612</pub-id>
          <pub-id pub-id-type="pii">S0140-6736(20)32342-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Charles</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Pozo-Rosich</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Targeting calcitonin gene–related peptide: a new era in migraine therapy</article-title>
          <source>Lancet</source>
          <year>2019</year>
          <volume>394</volume>
          <issue>10210</issue>
          <fpage>1765</fpage>
          <lpage>1774</lpage>
          <pub-id pub-id-type="doi">10.1016/S0140-6736(19)32504-8</pub-id>
          <pub-id pub-id-type="medline">31668411</pub-id>
          <pub-id pub-id-type="pii">S0140-6736(19)32504-8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ornello</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Casalena</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Frattale</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Gabriele</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Affaitati</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Giamberardino</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Assetta</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Maddestra</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Marzoli</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Viola</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Cerone</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Marini</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Pistoia</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Sacco</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Real-life data on the efficacy and safety of erenumab in the Abruzzo region, central Italy</article-title>
          <source>J Headache Pain</source>
          <year>2020</year>
          <volume>21</volume>
          <issue>1</issue>
          <fpage>32</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://thejournalofheadacheandpain.biomedcentral.com/articles/10.1186/s10194-020-01102-9"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s10194-020-01102-9</pub-id>
          <pub-id pub-id-type="medline">32264820</pub-id>
          <pub-id pub-id-type="pii">10.1186/s10194-020-01102-9</pub-id>
          <pub-id pub-id-type="pmcid">PMC7137484</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ruff</surname>
              <given-names>DD</given-names>
            </name>
            <name name-style="western">
              <surname>Ford</surname>
              <given-names>JH</given-names>
            </name>
            <name name-style="western">
              <surname>Tockhorn-Heidenreich</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Stauffer</surname>
              <given-names>VL</given-names>
            </name>
            <name name-style="western">
              <surname>Govindan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Aurora</surname>
              <given-names>SK</given-names>
            </name>
            <name name-style="western">
              <surname>Terwindt</surname>
              <given-names>GM</given-names>
            </name>
            <name name-style="western">
              <surname>Goadsby</surname>
              <given-names>PJ</given-names>
            </name>
          </person-group>
          <article-title>Efficacy of galcanezumab in patients with episodic migraine and a history of preventive treatment failure: results from two global randomized clinical trials</article-title>
          <source>Eur J Neurol</source>
          <year>2020</year>
          <volume>27</volume>
          <issue>4</issue>
          <fpage>609</fpage>
          <lpage>618</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/31692188"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/ene.14114</pub-id>
          <pub-id pub-id-type="medline">31692188</pub-id>
          <pub-id pub-id-type="pmcid">PMC7155018</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Tepper</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Brandes</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Reuter</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Boudreau</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Dolezil</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Cheng</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Lenz</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Klatt</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Mikol</surname>
              <given-names>DD</given-names>
            </name>
          </person-group>
          <article-title>Efficacy and safety of erenumab (AMG334) in chronic migraine patients with prior preventive treatment failure: a subgroup analysis of a randomized, double-blind, placebo-controlled study</article-title>
          <source>Cephalalgia</source>
          <year>2018</year>
          <volume>38</volume>
          <issue>10</issue>
          <fpage>1611</fpage>
          <lpage>1621</lpage>
          <pub-id pub-id-type="doi">10.1177/0333102418788347</pub-id>
          <pub-id pub-id-type="medline">29984601</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ferrari</surname>
              <given-names>MD</given-names>
            </name>
            <name name-style="western">
              <surname>Diener</surname>
              <given-names>HC</given-names>
            </name>
            <name name-style="western">
              <surname>Ning</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Galic</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Cohen</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Mueller</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ahn</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Schwartz</surname>
              <given-names>YC</given-names>
            </name>
            <name name-style="western">
              <surname>Grozinski-Wolff</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Janka</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Fremanezumab versus placebo for migraine prevention in patients with documented failure to up to four migraine preventive medication classes (FOCUS): a randomised, double-blind, placebo-controlled, phase 3b trial</article-title>
          <source>Lancet</source>
          <year>2019</year>
          <volume>394</volume>
          <issue>10203</issue>
          <fpage>1030</fpage>
          <lpage>1040</lpage>
          <pub-id pub-id-type="doi">10.1016/S0140-6736(19)31946-4</pub-id>
          <pub-id pub-id-type="medline">31427046</pub-id>
          <pub-id pub-id-type="pii">S0140-6736(19)31946-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="web">
          <article-title>Guideline on good pharmacovigilance practices (GVP)</article-title>
          <source>European Medicines Agency</source>
          <year>2017</year>
          <access-date>2021-10-03</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ema.europa.eu/en/documents/regulatory-procedural-guideline/guideline-good-pharmacovigilance-practices-gvp-module-vi-collection-management-submission-reports_en.pdf">https://www.ema.europa.eu/en/documents/regulatory-procedural-guideline/guideline-good-pharmacovigilance-practices-gvp-module-vi-collection-management-submission-reports_en.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pappa</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Stergioulas</surname>
              <given-names>LK</given-names>
            </name>
          </person-group>
          <article-title>Harnessing social media data for pharmacovigilance: a review of current state of the art, challenges and future directions</article-title>
          <source>Int J Data Sci Anal</source>
          <year>2019</year>
          <volume>8</volume>
          <issue>2</issue>
          <fpage>113</fpage>
          <lpage>135</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1007/s41060-019-00175-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s41060-019-00175-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <collab>GBD 2016 Headache Collaborators</collab>
          </person-group>
          <article-title>Global, regional, and national burden of migraine and tension-type headache, 1990-2016: a systematic analysis for the Global Burden of Disease study 2016</article-title>
          <source>Lancet Neurol</source>
          <year>2018</year>
          <volume>17</volume>
          <issue>11</issue>
          <fpage>954</fpage>
          <lpage>976</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1474-4422(18)30322-3"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S1474-4422(18)30322-3</pub-id>
          <pub-id pub-id-type="medline">30353868</pub-id>
          <pub-id pub-id-type="pii">S1474-4422(18)30322-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC6191530</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Saffi</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Do</surname>
              <given-names>TP</given-names>
            </name>
            <name name-style="western">
              <surname>Hansen</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Dodick</surname>
              <given-names>DW</given-names>
            </name>
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The migraine landscape on YouTube: a review of YouTube as a source of information on migraine</article-title>
          <source>Cephalalgia</source>
          <year>2020</year>
          <volume>40</volume>
          <issue>12</issue>
          <fpage>1363</fpage>
          <lpage>1369</lpage>
          <pub-id pub-id-type="doi">10.1177/0333102420943891</pub-id>
          <pub-id pub-id-type="medline">32703016</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bojazar</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Do</surname>
              <given-names>TP</given-names>
            </name>
            <name name-style="western">
              <surname>Hansen</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Dodick</surname>
              <given-names>DW</given-names>
            </name>
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Googling migraine: a study of Google as an information resource of migraine management</article-title>
          <source>Cephalalgia</source>
          <year>2020</year>
          <volume>40</volume>
          <issue>14</issue>
          <fpage>1633</fpage>
          <lpage>1644</lpage>
          <pub-id pub-id-type="doi">10.1177/0333102420942241</pub-id>
          <pub-id pub-id-type="medline">32703015</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Viana</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hougaard</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Do</surname>
              <given-names>TP</given-names>
            </name>
          </person-group>
          <article-title>The depiction of migraine visual aura on the internet</article-title>
          <source>Cephalalgia</source>
          <year>2020</year>
          <volume>40</volume>
          <issue>10</issue>
          <fpage>1113</fpage>
          <lpage>1118</lpage>
          <pub-id pub-id-type="doi">10.1177/0333102420927027</pub-id>
          <pub-id pub-id-type="medline">32460539</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Reddit’s 2019 Year in Review</article-title>
          <source>Reddit Blog</source>
          <access-date>2020-09-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://redditblog.com/2019/12/04/reddits-2019-year-in-review/">https://redditblog.com/2019/12/04/reddits-2019-year-in-review/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lijffijt</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Papapetrou</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Puolamäki</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Heikki</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Gunopulos</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Hofmann</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Malerba</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Vazirgiannis</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Analyzing word frequencies in large text corpora using inter-arrival times and bootstrapping</article-title>
          <source>Machine Learning and Knowledge Discovery in Databases, Part II</source>
          <year>2011</year>
          <publisher-loc>Heidelberg, Germany</publisher-loc>
          <publisher-name>Springer</publisher-name>
          <fpage>341</fpage>
          <lpage>357</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Baumgartner</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zannettou</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Keegan</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Squire</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Blackburn</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>The Pushshift Reddit Dataset</article-title>
          <year>2020</year>
          <conf-name>Proceedings of the International AAAI Conference on Web and Social Media</conf-name>
          <conf-date>May 26, 2020</conf-date>
          <conf-loc>New York, NY</conf-loc>
          <fpage>830</fpage>
          <lpage>839</lpage>
          <pub-id pub-id-type="doi">10.1609/icwsm.v14i1.7347</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Charles</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Rapoport</surname>
              <given-names>AM</given-names>
            </name>
          </person-group>
          <article-title>The American Headache Society's position statement on integrating new migraine treatments into clinical practice—comments</article-title>
          <source>Headache</source>
          <year>2019</year>
          <volume>59</volume>
          <issue>4</issue>
          <fpage>629</fpage>
          <pub-id pub-id-type="doi">10.1111/head.13496</pub-id>
          <pub-id pub-id-type="medline">30973200</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lijffijt</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Nevalainen</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Säily</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Papapetrou</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Puolamäki</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Mannila</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Significance testing of word frequencies in corpora</article-title>
          <source>Digit Scholarsh Humanit</source>
          <year>2014</year>
          <volume>31</volume>
          <issue>2</issue>
          <fpage>374</fpage>
          <lpage>397</lpage>
          <pub-id pub-id-type="doi">10.1093/llc/fqu064</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Linde</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Mulleners</surname>
              <given-names>WM</given-names>
            </name>
            <name name-style="western">
              <surname>Chronicle</surname>
              <given-names>EP</given-names>
            </name>
            <name name-style="western">
              <surname>McCrory</surname>
              <given-names>DC</given-names>
            </name>
          </person-group>
          <article-title>Topiramate for the prophylaxis of episodic migraine in adults</article-title>
          <source>Cochrane Database Syst Rev</source>
          <year>2013</year>
          <volume>2013</volume>
          <issue>6</issue>
          <fpage>CD010610</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/23797676"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/14651858.CD010610</pub-id>
          <pub-id pub-id-type="medline">23797676</pub-id>
          <pub-id pub-id-type="pmcid">PMC7388931</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Linde</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Rossnagel</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Propranolol for migraine prophylaxis</article-title>
          <source>Cochrane Database Syst Rev</source>
          <year>2004</year>
          <fpage>CD003225</fpage>
          <pub-id pub-id-type="doi">10.1002/14651858.CD003225.pub2</pub-id>
          <pub-id pub-id-type="medline">15106196</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lattanzi</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Brigo</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Trinka</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Vernieri</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Corradetti</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Dobran</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Silvestrini</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Erenumab for preventive treatment of migraine: a systematic review and meta-analysis of efficacy and safety</article-title>
          <source>Drugs</source>
          <year>2019</year>
          <volume>79</volume>
          <issue>4</issue>
          <fpage>417</fpage>
          <lpage>431</lpage>
          <pub-id pub-id-type="doi">10.1007/s40265-019-01069-1</pub-id>
          <pub-id pub-id-type="medline">30793254</pub-id>
          <pub-id pub-id-type="pii">10.1007/s40265-019-01069-1</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Do</surname>
              <given-names>TP</given-names>
            </name>
            <name name-style="western">
              <surname>Younis</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Ashina</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <person-group person-group-type="editor">
            <name name-style="western">
              <surname>Maassen van den Brink</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Martelletti</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Erenumab</article-title>
          <source>Monoclonal Antibodies in Headache</source>
          <year>2021</year>
          <publisher-loc>Cham, Switzerland</publisher-loc>
          <publisher-name>Springer</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <collab>Amgen Inc</collab>
          </person-group>
          <article-title>Aimovig (erenumab-aooe) [package insert]</article-title>
          <source>U.S. Food and Drug Administration</source>
          <year>2018</year>
          <access-date>2020-09-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.accessdata.fda.gov/drugsatfda_docs/label/2018/761077s000lbl.pdf">https://www.accessdata.fda.gov/drugsatfda_docs/label/2018/761077s000lbl.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lambru</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Hill</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Tylova</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Andreou</surname>
              <given-names>AP</given-names>
            </name>
          </person-group>
          <article-title>A prospective real-world analysis of erenumab in refractory chronic migraine</article-title>
          <source>J Headache Pain</source>
          <year>2020</year>
          <volume>21</volume>
          <issue>1</issue>
          <fpage>61</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://thejournalofheadacheandpain.biomedcentral.com/articles/10.1186/s10194-020-01127-0"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s10194-020-01127-0</pub-id>
          <pub-id pub-id-type="medline">32487102</pub-id>
          <pub-id pub-id-type="pii">10.1186/s10194-020-01127-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC7268737</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Aamodt</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Stovner</surname>
              <given-names>LJ</given-names>
            </name>
            <name name-style="western">
              <surname>Hagen</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Zwart</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>Comorbidity of headache and gastrointestinal complaints. The Head-HUNT Study</article-title>
          <source>Cephalalgia</source>
          <year>2008</year>
          <volume>28</volume>
          <issue>2</issue>
          <fpage>144</fpage>
          <lpage>151</lpage>
          <pub-id pub-id-type="doi">10.1111/j.1468-2982.2007.01486.x</pub-id>
          <pub-id pub-id-type="medline">18197884</pub-id>
          <pub-id pub-id-type="pii">CHA1486</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Martami</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Ghorbani</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Abolhasani</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Togha</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Meysamie</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sharifi</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Razeghi Jahromi</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Comorbidity of gastrointestinal disorders, migraine, and tension-type headache: a cross-sectional study in Iran</article-title>
          <source>Neurol Sci</source>
          <year>2018</year>
          <volume>39</volume>
          <issue>1</issue>
          <fpage>63</fpage>
          <lpage>70</lpage>
          <pub-id pub-id-type="doi">10.1007/s10072-017-3141-0</pub-id>
          <pub-id pub-id-type="medline">29022143</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10072-017-3141-0</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Falkenberg</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Bjerg</surname>
              <given-names>HR</given-names>
            </name>
            <name name-style="western">
              <surname>Olesen</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Two-hour CGRP infusion causes gastrointestinal hyperactivity: possible relevance for CGRP antibody treatment</article-title>
          <source>Headache</source>
          <year>2020</year>
          <volume>60</volume>
          <issue>5</issue>
          <fpage>929</fpage>
          <lpage>937</lpage>
          <pub-id pub-id-type="doi">10.1111/head.13795</pub-id>
          <pub-id pub-id-type="medline">32227602</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <collab>Eli Lilly and Company</collab>
          </person-group>
          <article-title>Emgality (galcanezumab) [summary of product characteristics]</article-title>
          <source>European Medicines Agency</source>
          <access-date>2020-09-29</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ema.europa.eu/en/documents/product-information/emgality-epar-product-information_en.pdf">https://www.ema.europa.eu/en/documents/product-information/emgality-epar-product-information_en.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Avery</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Anderson</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Bond</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Fortnum</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Gifford</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hannaford</surname>
              <given-names>PC</given-names>
            </name>
            <name name-style="western">
              <surname>Hazell</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Krska</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>McLernon</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Shakir</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Watson</surname>
              <given-names>MC</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of patient reporting of adverse drug reactions to the UK 'Yellow Card Scheme': literature review, descriptive and qualitative analyses, and questionnaire surveys</article-title>
          <source>Health Technol Assess</source>
          <year>2011</year>
          <volume>15</volume>
          <issue>20</issue>
          <fpage>1</fpage>
          <lpage>234, iii-iv</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.3310/hta15200"/>
          </comment>
          <pub-id pub-id-type="doi">10.3310/hta15200</pub-id>
          <pub-id pub-id-type="medline">21545758</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bousquet</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Dahamna</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Guillemin-Lanne</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Darmoni</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Faviez</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Huot</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Katsahian</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Leroux</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Pereira</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Richard</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Schück</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Souvignet</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Lillo-Le Louët</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Texier</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>The Adverse Drug Reactions from Patient Reports in Social Media project: five major challenges to overcome to operationalize analysis and efficiently support pharmacovigilance process</article-title>
          <source>JMIR Res Protoc</source>
          <year>2017</year>
          <volume>6</volume>
          <issue>9</issue>
          <fpage>e179</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchprotocols.org/2017/9/e179/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/resprot.6463</pub-id>
          <pub-id pub-id-type="medline">28935617</pub-id>
          <pub-id pub-id-type="pii">v6i9e179</pub-id>
          <pub-id pub-id-type="pmcid">PMC5629348</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
