<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<?covid-19-tdm?>
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JFR</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Form Res</journal-id>
      <journal-title>JMIR Formative Research</journal-title>
      <issn pub-type="epub">2561-326X</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v7i1e38430</article-id>
      <article-id pub-id-type="pmid">36961787</article-id>
      <article-id pub-id-type="doi">10.2196/38430</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>A Personalized Avatar-Based Web Application to Help People Understand How Social Distancing Can Reduce the Spread of COVID-19: Cross-sectional, Observational, Pre-Post Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Mavragani</surname>
            <given-names>Amaryllis</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Javidan</surname>
            <given-names>Arshia</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Moloney</surname>
            <given-names>Max</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Etienne</surname>
            <given-names>Doriane</given-names>
          </name>
          <degrees>MSc, MBA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>VITAM – Centre de recherche en santé durable</institution>
            <institution>Université Laval</institution>
            <addr-line>2480, Chemin de la Canardière</addr-line>
            <addr-line>Québec, QC, G1J 2G1</addr-line>
            <country>Canada</country>
            <phone>1 418 663 5313 ext 12286</phone>
            <email>doriane.etienne.1@ulaval.ca</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-2414-2591</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Archambault</surname>
            <given-names>Patrick</given-names>
          </name>
          <degrees>MD, MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff2" ref-type="aff">2</xref>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-5090-6439</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Aziaka</surname>
            <given-names>Donovan</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-6209-0545</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Chipenda-Dansokho</surname>
            <given-names>Selma</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1979-5395</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Dubé</surname>
            <given-names>Eve</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1336-1510</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Fallon</surname>
            <given-names>Catherine S</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4525-4066</ext-link>
        </contrib>
        <contrib id="contrib7" contrib-type="author">
          <name name-style="western">
            <surname>Hakim</surname>
            <given-names>Hina</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0517-2621</ext-link>
        </contrib>
        <contrib id="contrib8" contrib-type="author">
          <name name-style="western">
            <surname>Kindrachuk</surname>
            <given-names>Jason</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3305-7084</ext-link>
        </contrib>
        <contrib id="contrib9" contrib-type="author">
          <name name-style="western">
            <surname>Krecoum</surname>
            <given-names>Dan</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-0868-6150</ext-link>
        </contrib>
        <contrib id="contrib10" contrib-type="author">
          <name name-style="western">
            <surname>MacDonald</surname>
            <given-names>Shannon E</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff6" ref-type="aff">6</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4675-4433</ext-link>
        </contrib>
        <contrib id="contrib11" contrib-type="author">
          <name name-style="western">
            <surname>Ndjaboue</surname>
            <given-names>Ruth</given-names>
          </name>
          <degrees>MA, MPH, PhD</degrees>
          <xref rid="aff8" ref-type="aff">8</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-4716-6505</ext-link>
        </contrib>
        <contrib id="contrib12" contrib-type="author">
          <name name-style="western">
            <surname>Noubi</surname>
            <given-names>Magniol</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3040-0326</ext-link>
        </contrib>
        <contrib id="contrib13" contrib-type="author">
          <name name-style="western">
            <surname>Paquette</surname>
            <given-names>Jean-Sébastien</given-names>
          </name>
          <degrees>MD, MSc</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9524-6761</ext-link>
        </contrib>
        <contrib id="contrib14" contrib-type="author">
          <name name-style="western">
            <surname>Parent</surname>
            <given-names>Elizabeth</given-names>
          </name>
          <degrees>MA</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8332-9538</ext-link>
        </contrib>
        <contrib id="contrib15" contrib-type="author">
          <name name-style="western">
            <surname>Witteman</surname>
            <given-names>Holly O</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff3" ref-type="aff">3</xref>
          <xref rid="aff8" ref-type="aff">9</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4192-0682</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>VITAM – Centre de recherche en santé durable</institution>
        <institution>Université Laval</institution>
        <addr-line>Québec, QC</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Emergency Medicine</institution>
        <institution>Centre intégré de santé et de services sociaux de Chaudière-Appalaches</institution>
        <addr-line>Québec, QC</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Family Medicine and Emergency Medicine</institution>
        <institution>Faculty of Medicine</institution>
        <institution>Université Laval</institution>
        <addr-line>Québec, QC</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Department of Anthropology</institution>
        <institution>Université Laval</institution>
        <addr-line>Québec, QC</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>Laboratory of Emerging Viruses</institution>
        <institution>Department of Medical Microbiology &#38; Infectious Diseases</institution>
        <institution>University of Manitoba</institution>
        <addr-line>Winnipeg, MB</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff6">
        <label>6</label>
        <institution>Faculty of Nursing &#38; School of Public Health</institution>
        <institution>University of Alberta</institution>
        <addr-line>Edmonton, AB</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff7">
        <label>7</label>
        <institution>Research Center on Aging</institution>
        <institution>Centre intégré universitaire de santé et de services sociaux de l'Estrie - CHU de Sherbrooke</institution>
        <institution>Université de Sherbrooke</institution>
        <addr-line>Sherbrooke, QC</addr-line>
        <country>Canada</country>
      </aff>
      <aff id="aff8">
        <label>8</label>
        <institution>Research Centre of the CHU de Québec</institution>
        <institution>Laval University</institution>
        <addr-line>Québec, QC</addr-line>
        <country>Canada</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Doriane Etienne <email>doriane.etienne.1@ulaval.ca</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <year>2023</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>25</day>
        <month>4</month>
        <year>2023</year>
      </pub-date>
      <volume>7</volume>
      <elocation-id>e38430</elocation-id>
      <history>
        <date date-type="received">
          <day>1</day>
          <month>4</month>
          <year>2022</year>
        </date>
        <date date-type="rev-request">
          <day>19</day>
          <month>10</month>
          <year>2022</year>
        </date>
        <date date-type="rev-recd">
          <day>28</day>
          <month>2</month>
          <year>2023</year>
        </date>
        <date date-type="accepted">
          <day>1</day>
          <month>3</month>
          <year>2023</year>
        </date>
      </history>
      <copyright-statement>©Doriane Etienne, Patrick Archambault, Donovan Aziaka, Selma Chipenda-Dansokho, Eve Dubé, Catherine S Fallon, Hina Hakim, Jason Kindrachuk, Dan Krecoum, Shannon E MacDonald, Ruth Ndjaboue, Magniol Noubi, Jean-Sébastien Paquette, Elizabeth Parent, Holly O Witteman. Originally published in JMIR Formative Research (https://formative.jmir.org), 25.04.2023.</copyright-statement>
      <copyright-year>2023</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Formative Research, is properly cited. The complete bibliographic information, a link to the original publication on https://formative.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://formative.jmir.org/2023/1/e38430" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>To reduce the transmission of SARS-CoV-2 and the associated spread of COVID-19, many jurisdictions around the world imposed mandatory or recommended social or physical distancing. As a result, at the beginning of the pandemic, various communication materials appeared online to promote distancing. Explanations of the science underlying these mandates or recommendations were either highly technical or highly simplified.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aimed to understand the effects of a dynamic visualization on distancing. Our overall aim was to help people understand the dynamics of the spread of COVID-19 in their community and the implications of their own behavior for themselves, those around them, the health care system, and society.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>Using Scrum, which is an agile framework; JavaScript (Vue.js framework); and code already developed for risk communication in another context of infectious disease transmission, we rapidly developed a new personalized web application. In our application, people make avatars that represent themselves and the people around them. These avatars are integrated into a 3-minute animation illustrating an epidemiological model for COVID-19 transmission, showing the differences in transmission with and without distancing. During the animation, the narration explains the science of how distancing reduces the transmission of COVID-19 in plain language in English or French. The application offers full captions to complement the narration and a descriptive transcript for people using screen readers. We used Google Analytics to collect standard usage statistics. A brief, anonymous, optional survey also collected self-reported distancing behaviors and intentions in the previous and coming weeks, respectively. We launched and disseminated the application on Twitter and Facebook on April 8, 2020, and April 9, 2020.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>After 26 days, the application received 3588 unique hits from 82 countries. The optional survey at the end of the application collected 182 responses. Among this small subsample of users, survey respondents were nearly (170/177, 96%) already practicing distancing and indicated that they intended to practice distancing in the coming week (172/177, 97.2%). Among the small minority of people (n=7) who indicated that they had not been previously practicing distancing, 2 (29%) reported that they would practice distancing in the week to come.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>We developed a web application to help people understand the relationship between individual-level behavior and population-level effects in the context of an infectious disease spread. This study also demonstrates how agile development can be used to quickly create personalized risk messages for public health issues like a pandemic. The nonrandomized design of this rapid study prevents us from concluding the application’s effectiveness; however, results thus far suggest that avatar-based visualizations may help people understand their role in infectious disease transmission.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>social distancing</kwd>
        <kwd>COVID-19</kwd>
        <kwd>SARS-CoV-2</kwd>
        <kwd>pandemic</kwd>
        <kwd>personalized risk communication</kwd>
        <kwd>public health</kwd>
        <kwd>digital health intervention</kwd>
        <kwd>web application</kwd>
        <kwd>visualization</kwd>
        <kwd>personalized avatar</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Around the world, COVID-19 responses included messaging and interventions in digital media spaces [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. To reduce the transmission of SARS-CoV-2 and the associated spread of COVID-19, many jurisdictions around the world imposed mandatory or recommended social or physical distancing (hereafter, “distancing”) [<xref ref-type="bibr" rid="ref3">3</xref>]. Distancing is a measure that helps reduce the spread of highly contagious respiratory viruses when a potential transmission-reducing vaccine is not available. Distancing not only limits contact or the frequency of gathering in public places [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref6">6</xref>] but also reduces the increase in the number of cases in a community, thereby reducing or controlling the potential burden on health systems [<xref ref-type="bibr" rid="ref7">7</xref>]. As a result, at the beginning of the pandemic, various communication materials appeared online to promote distancing [<xref ref-type="bibr" rid="ref8">8</xref>].</p>
      <p>Implementing distancing is only possible with public cooperation [<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref10">10</xref>]. Such cooperation requires that people understand what they are being asked to do and why so that they adopt appropriate behaviors to help manage the pandemic [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. Effective public communication in times of crisis should be balanced to reduce negative factors such as anxiety or misinformation while promoting cooperation with recommendations [<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref14">14</xref>]. Public health messages must also be tailored to address public concerns and speak to all members of the public across diverse contexts, backgrounds, and levels of understanding [<xref ref-type="bibr" rid="ref15">15</xref>]. A rapid systematic review of the determinants of preventive behavior for viruses similar to COVID-19 found small positive correlations between perceptions of severity, threat, efficacy, and the adoption of distancing behavior. However, for those concerned, anxious, or worried about their relatives and themselves being ill, the correlation between the adoption of distancing behavior and emotions was moderate to strong. The evidence from this review suggests that a focus on awareness of the impact of COVID-19 on the individual or their surroundings may assist in promoting distancing [<xref ref-type="bibr" rid="ref12">12</xref>].</p>
      <p>With these considerations in mind, at the beginning of the COVID-19 pandemic, we observed that early digital risk communication materials shared on social media explaining the science of distancing either introduced distancing in a highly technical way or in an overly simplified way [<xref ref-type="bibr" rid="ref16">16</xref>]. There appeared to be fewer communication materials in the middle ground between these extremes that would be suitable across levels of health literacy to help promote distancing [<xref ref-type="bibr" rid="ref17">17</xref>]. Hart and colleagues [<xref ref-type="bibr" rid="ref18">18</xref>] described how the following 3 behavioral science levers may promote distancing: (1) demonstrate: show what distancing is by explaining what to do and making it memorable; (2) enable: make distancing easy, and offer reliable sources of information that address recommendations; (3) motivate and prompt: provide information about the consequences of distancing or of catching COVID-19.</p>
      <p>We therefore aimed to create a digital risk communication tool using these levers. Specifically, we adapted a personalized avatar-based web application to show the impact of distancing on COVID-19 transmission in a person’s community. Our study extends previous research by Hakim and colleagues [<xref ref-type="bibr" rid="ref19">19</xref>], who developed a web application to communicate epidemiological evidence on herd immunity through an animated visualization using personalized avatars. Designed to optimize users' cognitive and emotional responses, this application demonstrated the potential to communicate the relationship between individual behavior and community health. In this animation, the user builds a personalized avatar and 8 other avatars representing family, friends, or colleagues. The application then integrates these avatars into a 2-minute visualization showing how different parameters (eg, vaccine coverage, contacts within the community) influence herd immunity. The application aims to incorporate emotional considerations around infectious disease transmission, such as concern for people around oneself who may be more vulnerable to infection or severe outcomes [<xref ref-type="bibr" rid="ref19">19</xref>].</p>
      <p>Throughout the pandemic, the availability of clear and transparent information on the evolution of the scientific evidence has proved critical for building (or damaging) trust in institutions and empowering the public to better understand the situation and make evidence-based decisions. This cross-sectional, observational, pre-post study therefore aimed to understand the impact of a dynamic visualization on distancing during the early months of the COVID-19 pandemic. Our overall aim was to help people understand the dynamics of the spread of COVID-19 in their community and the implications of their own behavior for themselves, those around them, the health care system, and society. This paper describes the process of developing the web application and the associated study examining its effects on distancing and its usage metrics.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Storyboard and Script</title>
        <p>Based on a previous herd immunity or community immunity web application [<xref ref-type="bibr" rid="ref19">19</xref>], our multidisciplinary team developed a storyboard and narration script to explain distancing in both English and French (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
      </sec>
      <sec>
        <title>User Flow</title>
        <p>To build our web application, we started by modeling the new navigation flow (<xref rid="figure1" ref-type="fig">Figure 1</xref>) [<xref ref-type="bibr" rid="ref20">20</xref>]. First, a landing page informs users that the web application is optimized for the majority of browsers, except for Internet Explorer and Microsoft Edge, which are not supported. Users are directed to the home page, where they can select their preferred language (English or French) and access a descriptive transcript for people using screen readers (eg, people who are blind or vision-impaired) or with low bandwidth (eg, people who live in locations with limited internet service.) Next, users are directed to the avatar creation page. Once their avatar is created, users can launch a narrated animation featuring their personalized avatar. At the end of the animation, users receive additional information with links to Canadian government and international websites about COVID-19 prevention and are invited to complete a brief survey.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>User flow representation.</p>
          </caption>
          <graphic xlink:href="formative_v7i1e38430_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Web Application Development</title>
        <p>To develop our web application, we used Scrum, which is an agile framework [<xref ref-type="bibr" rid="ref21">21</xref>-<xref ref-type="bibr" rid="ref23">23</xref>], and JavaScript (Vue.js framework) as our programming language [<xref ref-type="bibr" rid="ref24">24</xref>]. Agile practices may be more pragmatic than waterfall software development methods, as they promote adaptive planning, evolutionary development, rapid delivery, and continuous improvement [<xref ref-type="bibr" rid="ref25">25</xref>-<xref ref-type="bibr" rid="ref27">27</xref>]. In keeping with our agile approach, we performed short iterations that allowed us to quickly adapt the prototype, adding new features and fixing bugs. By building on existing code and using a rapid development approach, in 3 weeks, we created a web application that respected accessibility standards, including providing descriptive text for people who are blind or vision-impaired and captions for people who are deaf or hearing-impaired.</p>
      </sec>
      <sec>
        <title>The Web Application</title>
        <p>In our web application, people create avatars (<xref rid="figure2" ref-type="fig">Figure 2</xref>) that represent themselves and people around them by choosing among a set of avatars, then adjusting skin color, hair color, glasses, facial hair, and head covering accessories such as a hijab, turban, and caps, as desired.</p>
        <p>These avatars are used in a 3-minute animation illustrating an infectious disease epidemiological model of COVID-19 [<xref ref-type="bibr" rid="ref28">28</xref>], showing the differences in transmission with and without distancing (<xref rid="figure3" ref-type="fig">Figure 3</xref>). During the animation, the narration explains the science of how distancing reduces the transmission of COVID-19 in plain language. The web application offers complete captions for those who may be unable to hear the narration or who prefer to read it and a full written descriptive version for people who may be unable to see the visualization, for example, people using screen readers. At the end of the animation, we offered viewers a summary page on distancing with links to the websites of the Public Health Agency of Canada (COVID-19 Prevention, Quarantine, and Isolation) [<xref ref-type="bibr" rid="ref29">29</xref>] and the World Health Organization for more information [<xref ref-type="bibr" rid="ref8">8</xref>].</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Avatar creation screen.</p>
          </caption>
          <graphic xlink:href="formative_v7i1e38430_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>Differences in transmission (A) without distancing and (B) with distancing.</p>
          </caption>
          <graphic xlink:href="formative_v7i1e38430_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>Evaluation Study Design and Data Collection</title>
        <p>This study was a cross-sectional, observational, pre-post study in which we asked people to self-report their behavior in the week before viewing the animation and their behavioral intentions for the week to come and collected usage metrics.</p>
        <sec>
          <title>Ethical Considerations</title>
          <p>This study received ethical authorization from the Research Ethics Committee of Laval University (approval number 2020-098/27-03-2020). All study participants were notified in written form about the aim of the study and the fact that their participation was voluntary. They were assured of the anonymity of their data. By completing the 5-question survey, participants gave their implied free and informed consent electronically to the use of their answers in the study. Study participants were not compensated.</p>
        </sec>
        <sec>
          <title>Survey</title>
          <p>After viewing the 3-minute animation, participants were asked to complete a short, anonymous, optional 5-question survey. Participants had learned about the application after we disseminated the web application on April 8,2020, and April 9, 2020, through social media platforms. We used the lab's Twitter and Facebook accounts (@wittemanlab) and the personal Twitter account of the principal investigator, Holly Witteman (@hwitteman). The link was subsequently shared by the Twitter accounts of the Canadian Institutes of Health Research, Jane Philpott (former Canada Minister of Health), and other influential accounts. We used a convenience sample for the survey and expected to recruit about 200 participants. To be eligible for inclusion, participants had to be 18 years of age or older, able to read and understand English or French, and able to provide informed consent electronically by continuing the survey. We excluded responses from people who indicated that they could not practice distancing (for example, because they were essential workers.) We asked participants to self-report the following short set of sociodemographic characteristics: year of birth, place of residence, gender, and level of education. We also used these variables as independent variables in our analyses. We collected survey responses via Qualtrics from April 8, 2020, to May 3, 2020.</p>
        </sec>
        <sec>
          <title>Application Usage</title>
          <p>To study how people interact and generate usage metrics, we linked the web application to Google Analytics [<xref ref-type="bibr" rid="ref30">30</xref>].</p>
        </sec>
      </sec>
      <sec>
        <title>Outcomes and Additional Data</title>
        <sec>
          <title>Survey</title>
          <p>Our primary outcome of interest was participants’ self-reported behavioral intentions to practice distancing in the week after viewing the web application compared with the previous week. We assessed each of these by asking participants to indicate their response on a 7-point, agree-disagree, Likert-type scale to the items, “In the past week (7 days), I practiced distancing,” and “In the coming week, I intend to practice distancing.” We included an option to indicate an inability to distance due to a public-facing occupation, “Not applicable (eg, I work in an essential service).”</p>
        </sec>
        <sec>
          <title>Application Usage</title>
          <p>From Google Analytics, we collected data about active users, sessions, technology use (device, operating system), language, and geographic location. According to the definition provided by Google, an active user is a user who has accessed the web application. At the first access of the web application, all data are stored and associated with each user. Specifically, a unique cookie stores a Google Analytics customer ID, which enables identification of the user on all devices they may use to reach the web application and to determine their number of connections [<xref ref-type="bibr" rid="ref31">31</xref>]. Google Analytics collects descriptive data about the technologies used (eg, mobile devices) by the study population. The tracking code also enables the collection of browser data, such as the configured language, browser (eg, Chrome, Safari, Firefox), device (eg, iPad, desktop), and operating system (eg, Windows, macOS) used to access the web application. These data are updated with user activity. For each user, a time period called a session begins when the user accesses a page containing the tracking code. A session ends after 30 minutes of inaction. If the user comes back later, a new session begins. Data on language and geographic location are obtained using the ISO 639 language code [<xref ref-type="bibr" rid="ref32">32</xref>] and ISO 3166 country code [<xref ref-type="bibr" rid="ref33">33</xref>] set in the user’s browser.</p>
        </sec>
      </sec>
      <sec>
        <title>Analyses</title>
        <sec>
          <title>Survey</title>
          <p>We compared participants’ behavioral intentions to practice distancing in the week to come with their self-reported distancing behavior in the previous week. Given the exploratory nature of this study, we conducted both a paired <italic>t</italic> test and a Wilcoxon signed-rank test with continuity correction. Then, using ANOVA, we tested the interactions between sociodemographic data and differences between distancing intentions and self-reported distancing in the previous week. We also performed additional comparisons to explore which modalities differed using the least-squares mean method (lsmeans) [<xref ref-type="bibr" rid="ref34">34</xref>]. We performed all statistical analyses in R, Version 3.6.2 [<xref ref-type="bibr" rid="ref35">35</xref>]</p>
        </sec>
        <sec>
          <title>Application Usage</title>
          <p>We descriptively analyzed user data generated by Google Analytics to understand how users interacted with the web application to identify potential future improvements [<xref ref-type="bibr" rid="ref30">30</xref>].</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Survey</title>
        <sec>
          <title>Population</title>
          <p>The optional survey at the end of the web application yielded 182 responses, representing 5.1% of all 3588 users. Overall, most respondents completed the optional survey in English (171/182, 94%). The average age of respondents was 42.7 (SD 15.0) years. Most respondents (111/182, 60.9%) identified as women, indicated they were university graduates (156/182, 85.7%), and reported living outside Canada (102/182, 56%; <xref ref-type="table" rid="table1">Table 1</xref>). Of the participants, 5 indicated they could not practice distancing; we conducted analyses on the remaining 177 respondents.</p>
          <table-wrap position="float" id="table1">
            <label>Table 1</label>
            <caption>
              <p>Characteristics of respondents to the optional survey (n=182).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="30"/>
              <col width="650"/>
              <col width="320"/>
              <thead>
                <tr valign="top">
                  <td colspan="2">Responses</td>
                  <td>Results</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td colspan="3">
                    <bold>User language, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>English</td>
                  <td>171 (94)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>French</td>
                  <td>11 (6)</td>
                </tr>
                <tr valign="top">
                  <td colspan="2">Age (years), mean (SD)<sup>a</sup></td>
                  <td>42.7 (15)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Place of residence, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Alberta (AB)</td>
                  <td>10 (5.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — British Columbia (BC)</td>
                  <td>5 (2.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Manitoba (MB)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — New Brunswick (NB)</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Newfoundland and Labrador (NL)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Northwest Territories (NT)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Nova Scotia (NS)</td>
                  <td>5 (2.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Nunavut (NU)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Ontario (ON)</td>
                  <td>44 (24.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Prince Edward Island (PE)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Quebec (QC)</td>
                  <td>11 (6.1)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Saskatchewan (SK)</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Canada — Yukon (YT)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Outside Canada</td>
                  <td>102 (56)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Prefer not to answer</td>
                  <td>3 (1.6)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Gender identity, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Female</td>
                  <td>111 (61)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Male</td>
                  <td>68 (37.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Indigenous or other cultural gender minority identity (eg, 2-spirit)</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Another gender (eg, non-binary, gender-fluid)</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Prefer not to answer</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Education level, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Some elementary school (completed or not)</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Some high school (not completed)</td>
                  <td>7 (3.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>High school diploma</td>
                  <td>9 (5.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Apprenticeship or trade certificate or diploma</td>
                  <td>3 (1.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>College or polytechnical school certificate or diploma</td>
                  <td>6 (3.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>University degree, bachelor level or below</td>
                  <td>66 (36.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>University graduate degree (Master’s level)</td>
                  <td>52 (28.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>University graduate degree (Doctorate level)</td>
                  <td>38 (20.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Do not know</td>
                  <td>0 (0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Prefer not to answer</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Distancing, n (%)<sup>b</sup></bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Practiced distancing in the past week</td>
                  <td>170 (96)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Did not practice distancing in the past week</td>
                  <td>7 (4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Intend to practice distancing in the week to come</td>
                  <td>172 (97.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>No intention to practice distancing in the week to come</td>
                  <td>5 (2.8)</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table1fn1">
                <p><sup>a</sup>n=174.</p>
              </fn>
              <fn id="table1fn2">
                <p><sup>b</sup>n=177.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
        <sec>
          <title>Behavioral Intentions Regarding Distancing</title>
          <p>Among the 177 respondents, 170 (96.0%) were already practicing distancing, and 172 (97.2%) intended to practice distancing in the coming week. Of those (n=7) not yet practicing distancing in the previous week, 2 (29%) indicated, after using the web application, that they would practice distancing in the week to come. None of the people already practicing distancing indicated that they intended to change that behavior.</p>
          <p>We found no significant difference between the respondents’ initial distancing practice (mean 7.50, SD 1.13) and intention to practice distancing in the coming week (mean 7.59, SD 1.05; t<sub>176</sub>= 1.87, <italic>P</italic>=.06; <xref ref-type="table" rid="table2">Table 2</xref>). The Wilcoxon signed-rank test (<italic>P</italic>=.08) was also not statistically significant.</p>
          <p>We found a statistically significant main effect of the place of residence (<italic>F</italic><sub>7,146</sub>=2.94, <italic>P</italic>=.006) and gender (<italic>F</italic><sub>1,146</sub>=3.86, <italic>P</italic>=.05) on the reported practice of distancing.</p>
          <p>We observed a statistically significant interaction between the place of residence “outside Canada” and the difference in reported practice (t<sub>146</sub>=2.653, <italic>P</italic>=.008), indicating the application had a stronger effect on intentions to practice distancing among people living outside Canada. We also observed a statistically significant interaction by self-reported gender. Specifically, the effect of the application was positive for female respondents (t<sub>146</sub>=0.61, <italic>P</italic>=.53) and negative for male respondents (t<sub>146</sub>=–0.49, <italic>P</italic>=.62; <xref rid="figure4" ref-type="fig">Figure 4</xref>).</p>
          <table-wrap position="float" id="table2">
            <label>Table 2</label>
            <caption>
              <p>Comparison of means between the initial and future practice of distancing reported after being exposed to the 3-minute animation, using a paired t test.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="280"/>
              <col width="260"/>
              <col width="160"/>
              <col width="300"/>
              <thead>
                <tr valign="top">
                  <td>Response</td>
                  <td>Results, mean (SD)</td>
                  <td><italic>t</italic> (df)</td>
                  <td><italic>P</italic> value</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Practiced distancing</td>
                  <td>7.50 (1.13)</td>
                  <td rowspan="2">1.87 (176)</td>
                  <td rowspan="2">.06</td>
                </tr>
                <tr valign="top">
                  <td>Intend to practice distancing</td>
                  <td>7.59 (1.05)</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
          <fig id="figure4" position="float">
            <label>Figure 4</label>
            <caption>
              <p>The least-squares mean related to gender interaction.</p>
            </caption>
            <graphic xlink:href="formative_v7i1e38430_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
      </sec>
      <sec>
        <title>Application Usage</title>
        <sec>
          <title>Users’ Characteristics and Use Pattern Data</title>
          <p>During the study period, from April 8, 2020, to May 3, 2020, we had a total of 3588 active users of the web application, of which 3349 users (93.3%) had a single session and 239 users (6.7%) had multiple sessions (<xref ref-type="table" rid="table3">Table 3</xref>). According to the ISO 3166 country code, the web application was used in 82 countries (<xref rid="figure5" ref-type="fig">Figure 5</xref>). The 3588 users were located in the Americas (2864/3588, 79.8%), Europe (411/3588, 11.5%), Oceania (166/3588, 4.6%), Asia (103/3588, 2.9%), and Africa (42/3588, 1.2%).</p>
          <p>Of the 3588 users, 3561 (93.3%) were new visitors, so they logged in only 1 time during the study period. Users who returned to the web application (“returning visitors”) numbered 254 (6.7%). This represents 3924 sessions, including 3349 (85.3%) single and 575 (14.7%) multiple sessions. Of these, the majority (1261/3924, 32.1%) of the sessions varied in duration from 181 seconds to 600 seconds, representing an average time spent on the web application of 3.18 minutes.</p>
          <table-wrap position="float" id="table3">
            <label>Table 3</label>
            <caption>
              <p>Users and use patterns (n=3588).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="30"/>
              <col width="30"/>
              <col width="520"/>
              <col width="0"/>
              <col width="420"/>
              <thead>
                <tr valign="top">
                  <td colspan="4">Use patterns</td>
                  <td>Results</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td colspan="5">
                    <bold>Active users n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="2">Single session users</td>
                  <td colspan="2">3349 (93.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="2">Multisession users</td>
                  <td colspan="2">239 (6.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="2">Survey respondents</td>
                  <td colspan="2">182 (5.1)</td>
                </tr>
                <tr valign="top">
                  <td colspan="5">
                    <bold>Geographic characteristics<sup>a</sup>, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Language</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>English</td>
                  <td colspan="2">3175 (88.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>French</td>
                  <td colspan="2">199 (5.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Both</td>
                  <td colspan="2">1 (0.1)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td colspan="2">213 (5.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Location</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>United States</td>
                  <td colspan="2">1830 (51.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Canada</td>
                  <td colspan="2">947 (26.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Australia</td>
                  <td colspan="2">149 (4.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>United Kingdom</td>
                  <td colspan="2">108 (3.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td colspan="2">554 (15.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Location by continent, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Americas</td>
                  <td colspan="2">2864 (79.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Europe</td>
                  <td colspan="2">411 (11.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Oceania</td>
                  <td colspan="2">166 (4.6)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Asia</td>
                  <td colspan="2">103 (2.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Africa</td>
                  <td colspan="2">42 (1.2)</td>
                </tr>
                <tr valign="top">
                  <td colspan="5">
                    <bold>Technology, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Device category</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Desktop</td>
                  <td colspan="2">1745 (48.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Cell phone</td>
                  <td colspan="2">1692 (47.1)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Tablet</td>
                  <td colspan="2">152 (4.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Browser</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Chrome</td>
                  <td colspan="2">1677 (46.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Safari</td>
                  <td colspan="2">1184 (33.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Safari (in-app)</td>
                  <td colspan="2">230 (6.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Firefox</td>
                  <td colspan="2">206 (5.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Android WebView</td>
                  <td colspan="2">150 (4.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Samsung Internet</td>
                  <td colspan="2">71 (2.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Edge</td>
                  <td colspan="2">29 (0.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Opera</td>
                  <td colspan="2">20 (0.6)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Mozilla Compatible Agent</td>
                  <td colspan="2">19 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Amazon Silk</td>
                  <td colspan="2">2 (0.1)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Operating system</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>iOS</td>
                  <td colspan="2">1145 (31.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Windows</td>
                  <td colspan="2">899 (25.1)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Macintosh</td>
                  <td colspan="2">788 (21.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Android</td>
                  <td colspan="2">697 (19.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Chrome OS</td>
                  <td colspan="2">37 (1.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Linux</td>
                  <td colspan="2">21 (0.6)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>BlackBerry</td>
                  <td colspan="2">2 (0.1)</td>
                </tr>
                <tr valign="top">
                  <td colspan="5">
                    <bold>Behavior</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="2">New visitors, n (%)</td>
                  <td colspan="2">3561 (93.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="2">Returning visitors, n (%)</td>
                  <td colspan="2">254 (6.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Count of sessions<sup>b</sup>, n (%)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Single session users</td>
                  <td colspan="2">3349 (85.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Multisession users (2 to 10 sessions)</td>
                  <td colspan="2">575 (14.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Session duration, (number of sessions)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>0-10 seconds</td>
                  <td colspan="2">1217</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>11-30 seconds</td>
                  <td colspan="2">220</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>31-60 seconds</td>
                  <td colspan="2">205</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>61-180 seconds</td>
                  <td colspan="2">790</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>181-600 seconds</td>
                  <td colspan="2">1261</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>601-1800 seconds</td>
                  <td colspan="2">224</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>≥1801 seconds</td>
                  <td colspan="2">7</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td colspan="4">
                    <bold>Session duration (minutes:seconds), mean</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>All users</td>
                  <td colspan="2">00:03:18</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Single session users</td>
                  <td colspan="2">00:03:27</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Multisession users</td>
                  <td colspan="2">00:02:23</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>New visitors</td>
                  <td colspan="2">00:03:26</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>
                    <break/>
                  </td>
                  <td>Returning visitors</td>
                  <td colspan="2">00:01:59</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table3fn1">
                <p><sup>a</sup>n=3588.</p>
              </fn>
              <fn id="table3fn2">
                <p><sup>b</sup>3924 sessions by 3588 users.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
          <fig id="figure5" position="float">
            <label>Figure 5</label>
            <caption>
              <p>Geographical location of users by country.</p>
            </caption>
            <graphic xlink:href="formative_v7i1e38430_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <p>In the behavioral flow for this web application, users have 5 points of interaction: landing page, home page, avatar creation, animation, and extra information with the optional survey (<xref rid="figure1" ref-type="fig">Figure 1</xref>). From the initial point of interaction (the landing page), of the 3924 sessions, 3690 (94%) sessions of total traffic arrived at the home page. From there, of the 3924 sessions, 3110 (79.2%) reached the avatar creation page, 2148 (54.7%) reached the animation page, and 681 (17.4%) reached the extra information with the optional survey page, meaning they had to have viewed the animation and fully completed the user's journey.</p>
          <p>These data enabled us to obtain information from participants' usage patterns, the journey through the web application, and to draw a user profile. The typical user had a browser set in English (3175/3588, 88.5%), was in the United States (1830/3588, 51.0%), and was mainly exposed to animation on desktop (1745/3588, 48.7%) from Chrome (1677/3588, 46.7%) or cell phone (1692/3588, 47.1%) with iOS (1145/3588, 31.9%) as the primary operating system. This new user logged in for a single session of at least one time for an average of 181 seconds, or an average of 3:26 minutes, and did not complete the optional survey on distancing.</p>
        </sec>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>The aim of this study was to understand the impact of a dynamic visualization on distancing during the early months of the COVID-19 pandemic. Our results led us to 3 principal observations, which we detail in the following paragraphs.</p>
        <p>First, given that the large majority of survey participants were already practicing distancing, we suggest that such communication materials might have dual roles. The first role is the more obvious one of helping to prompt behavior change. In our study, among the small subsample of survey respondents who reported not already practicing distancing, 29% indicated an intention to increase their practice of distancing in the coming week. Due to low numbers, this was not statistically significant. The second role may be to help reinforce existing desired behaviors. A substantial number of people went through the application in its entirety and responded to the survey at the end, indicating that they were already practicing distancing. Our survey did not include questions about why they might have spent this time, so we are unable to draw conclusions, but we hypothesize that, especially in the context of the early stages of a pandemic, people may seek reassurance that they are doing the right things. Public health communications may therefore offer information that is helpful in more subtle ways than prompting immediate behavior change. In other words, there may be a place for communications that are explicitly designed to reinforce existing behaviors and to be shared or shown to others, so that individuals can use them in their discussions with family, friends, and colleagues. Future studies should consider further exploring such sharing intentions.</p>
        <p>Second, we observe that characteristics such as gender and country of residence may influence the impact of digital public health communications. Our web application received 3588 unique hits from 82 countries after 26 days, having been shared on social media across Canada. The majority of survey respondents reported higher education levels. This may suggest that the application itself reached more people with higher levels of education, or it may reflect higher willingness to complete an optional survey among people with higher levels of education. We also noted that gender and place of residence interacted statistically significantly with distancing intentions. Specifically, among those living outside of Canada, participants who identified as women appeared to be impacted toward prosocial behavior (ie, distancing to protect oneself and others) following exposure to our intervention, while those who identified as men appeared to be impacted in an opposite direction.</p>
        <p>Third, usage analytics revealed that most people who abandoned the application gave up at the point at which they were asked to create 9 custom avatars prior to viewing the animation integrating those avatars. The rationale of custom avatars was to highlight the impact of distancing on people known to the user. However, creating 9 avatars may have been too much to ask. This suggests that, for future such applications, it will be important to find the right balance between personalization and the amount of information and time asked of the user.</p>
      </sec>
      <sec>
        <title>Comparison With Previous Literature</title>
        <p>As recommended by Gough and colleagues [<xref ref-type="bibr" rid="ref36">36</xref>], we had some success disseminating our message using Twitter. Although Twitter has since undergone changes and may no longer function in quite the same way as it did in 2020, social media remains an important vehicle for reaching a wide range of people. Indeed, during the COVID-19 pandemic, public health experts increasingly used social media to disseminate messages directly [<xref ref-type="bibr" rid="ref2">2</xref>]. To optimize the impact of evidence-based messages, academic and other institutional dissemination efforts may benefit from consultation, advice, or collaboration with social media experts.</p>
        <p>Our findings around potential differences in how people with varying identity characteristics received our intervention align with other studies showing that such characteristics matter and must be considered when designing public health messages [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref14">14</xref>,<xref ref-type="bibr" rid="ref37">37</xref>-<xref ref-type="bibr" rid="ref42">42</xref>]. As outlined by the World Health Organization in its strategy to combat COVID-19, triggering the right action requires delivering the right message to the right audience at the right time [<xref ref-type="bibr" rid="ref43">43</xref>].</p>
        <p>This research built on previous work by Hakim and colleagues [<xref ref-type="bibr" rid="ref19">19</xref>,<xref ref-type="bibr" rid="ref44">44</xref>] developing and evaluating an interactive visualization about how herd immunity or community immunity works. By incorporating avatars and considering the role of emotions, Hakim and colleagues [<xref ref-type="bibr" rid="ref19">19</xref>] aimed to personalize communication about infectious disease transmission. Our web application offers an adaptation of the original application, suggesting that such a visualization’s core functionality may be usable across related contexts [<xref ref-type="bibr" rid="ref45">45</xref>]. For example, although distancing is no longer the recommended avenue for reducing COVID-19 transmission, the visualization may be adapted to demonstrate the impact of mask wearing or vaccination on transmission. By linking our web application with Google Analytics, we obtained indirect feedback about user experience, technology usage, and most importantly, user pain points, enabling us and other teams building similar applications to better balance user burden and intervention effectiveness. Put plainly, a digital public health intervention is unlikely to be effective if people abandon it halfway through. According to our site usage metrics, we had 2 main throttle points: avatar creation and animation viewing. The avatar creation process (selecting an avatar, then customizing skin color, hair color, glasses, facial hair, and head coverings like hijabs, turbans, and caps) can take a long time, especially for new users. To address this issue, in future iterations building on this application, we will offer an option to automatically create additional avatars. Although no interaction was required in the animation viewing process, people might have abandoned it because it was too long, they didn’t wish to spend the time, they lacked the bandwidth to stream the video, or other reasons. Currently, we are refactoring this process to make it easier for future users. As a solution to this issue, we added the ability to move the visualization forward or backward and aim to use shorter animations.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>Our study had 4 main limitations. First, the nonrandomized study design limited our ability to draw firm conclusions about the actual effect of our web application on distancing. Second, the sample of survey respondents was smaller than the population of application users, with a total of 182 responses to the optional survey out of 3588 unique hits. In addition, given our recruitment methods and high baseline distancing practices, we likely observed selection bias in survey respondents. Third, we were able to measure distancing intentions but unable to measure self-reported behavior in a follow-up survey. Fourth and finally, the Google Analytics data limited our analyses. Such data are structured primarily for the needs of those analyzing marketing metrics such as acquisition, behavior, and conversion, not for scientific analyses. In future studies, we aim to explicitly link survey data with usage metrics for each individual study participant.</p>
      </sec>
      <sec>
        <title>Strengths</title>
        <p>This study has 3 main strengths. First, using agile development, we demonstrated the ability to adapt an existing application about herd immunity or community immunity, already optimized for users’ cognitive and emotional responses, to a new infectious disease context. This reinforces potential avenues for further research using already available software and approaches. Second, using social media showed that we could disseminate this type of material rapidly and reach a larger sample of the population than the convenience sample we had originally defined. Third and finally, this research provided insight on user experience to inform future applications designed to help members of the public understand the science of infectious disease transmission and how epidemiological evidence applies to them as individuals and to those around them.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>We developed a web application to help people understand the relationship between individual-level behavior and population-level effects in the context of infectious disease spread. This paper suggests that personalized risk communication, such as personalized avatar-based visualizations, could be effective. This study also demonstrates how agile development can be used to quickly create personalized risk messages for public health issues like a pandemic. The nonrandomized design of this rapid study does not allow conclusions about the web application’s effectiveness on distancing or knowledge retention. Future work may improve the visualization and involve more user testing.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Storyboard and script.</p>
        <media xlink:href="formative_v7i1e38430_app1.docx" xlink:title="DOCX File , 3640 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Data set that was generated and analyzed.</p>
        <media xlink:href="formative_v7i1e38430_app2.xlsx" xlink:title="XLSX File  (Microsoft Excel File), 23 KB"/>
      </supplementary-material>
    </app-group>
    <ack>
      <p>The authors gratefully acknowledge the contributions of Beate Sander and Noah M Ivers to this project as research associates and team members Aminata Camara, Sandrine Comeau, Excellent Kiyangi, Seyed Mostafa Mousavi, Charles Racine, and Gratianne Vaisson who contributed greatly to this research project but who were unable to accept the invitation to participate on this particular paper as co-authors due to time constraints. We thank Donavan Aziaka, Magniol Noubi, and Dan Krecoum for programming the web application. We thank all authors of the original articles who generously gave their time to validate the data we had extracted from their papers. We thank VITAM-The Sustainable Health Research Centre in Québec City, the Canadian Immunization Research Network (CIRN), and the Canadian Institutes of Health Research (CIHR). This study was funded by the CIHR (FDN-148426; principal investigator: HOW). The CIHR had no role in determining the study design, the plans for data collection or analysis, the decision to publish, or the preparation of this manuscript. Finally, we thank all study participants for helping us identify ways to improve education, communication, and decision-making in health care.</p>
    </ack>
    <notes>
      <sec>
        <title>Data Availability</title>
        <p>All data generated or analyzed during this study are included in this published article and are available in <xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>.</p>
      </sec>
    </notes>
    <fn-group>
      <fn fn-type="con">
        <p>DE, HOW, and ED contributed to the design of the study. DE, EP, and HOW contributed to data collection. DE conducted data analysis and interpretation. DE and HOW drafted the first version of the article with early revision by SCD, ED, CSF, HH, RN, and EP. PA, DA, SCD, ED, CSF, HH, JK, DK, SEM, RN, MN, JSP, EP, and HOW critically revised the article and approved the final version for submission for publication. DE and HOW had full access to all the data in the study and had final responsibility for the decision to submit for publication.</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Husain</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Galea</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Digital health interventions for COVID-19 in China: a retrospective analysis</article-title>
          <source>Intell Med</source>
          <year>2021</year>
          <month>05</month>
          <volume>1</volume>
          <issue>1</issue>
          <fpage>29</fpage>
          <lpage>36</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S2667-1026(21)00002-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.imed.2021.03.001</pub-id>
          <pub-id pub-id-type="medline">34447602</pub-id>
          <pub-id pub-id-type="pii">S2667-1026(21)00002-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC8079943</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Drescher</surname>
              <given-names>LS</given-names>
            </name>
            <name name-style="western">
              <surname>Roosen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Aue</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Dressel</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Schär</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Götz</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>The spread of COVID-19 crisis communication by German public authorities and experts on Twitter: quantitative content analysis</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2021</year>
          <month>12</month>
          <day>22</day>
          <volume>7</volume>
          <issue>12</issue>
          <fpage>e31834</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2021/12/e31834/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/31834</pub-id>
          <pub-id pub-id-type="medline">34710054</pub-id>
          <pub-id pub-id-type="pii">v7i12e31834</pub-id>
          <pub-id pub-id-type="pmcid">PMC8698804</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Qualls</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Levitt</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kanade</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wright-Jegede</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Dopson</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Biggerstaff</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Reed</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Uzicanin</surname>
              <given-names>A</given-names>
            </name>
            <collab>CDC Community Mitigation Guidelines Work Group</collab>
          </person-group>
          <article-title>Community mitigation guidelines to prevent pandemic influenza - United States, 2017</article-title>
          <source>MMWR Recomm Rep</source>
          <year>2017</year>
          <month>04</month>
          <day>21</day>
          <volume>66</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>34</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/28426646"/>
          </comment>
          <pub-id pub-id-type="doi">10.15585/mmwr.rr6601a1</pub-id>
          <pub-id pub-id-type="medline">28426646</pub-id>
          <pub-id pub-id-type="pmcid">PMC5837128</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mahtani</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Heneghan</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Aronson</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>What is the evidence for social distancing during global pandemics? A rapid summary of current knowledge</article-title>
          <source>The Centre for Evidence-Based Medicine</source>
          <access-date>2021-03-01</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.cebm.net/covid-19/what-is-the-evidence-for-social-distancing-during-global-pandemics/">https://www.cebm.net/covid-19/what-is-the-evidence-for-social-distancing-during-global-pandemics/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sun</surname>
              <given-names>KS</given-names>
            </name>
            <name name-style="western">
              <surname>Lau</surname>
              <given-names>TSM</given-names>
            </name>
            <name name-style="western">
              <surname>Yeoh</surname>
              <given-names>EK</given-names>
            </name>
            <name name-style="western">
              <surname>Chung</surname>
              <given-names>VCH</given-names>
            </name>
            <name name-style="western">
              <surname>Leung</surname>
              <given-names>YS</given-names>
            </name>
            <name name-style="western">
              <surname>Yam</surname>
              <given-names>CHK</given-names>
            </name>
            <name name-style="western">
              <surname>Hung</surname>
              <given-names>CT</given-names>
            </name>
          </person-group>
          <article-title>Effectiveness of different types and levels of social distancing measures: a scoping review of global evidence from earlier stage of COVID-19 pandemic</article-title>
          <source>BMJ Open</source>
          <year>2022</year>
          <month>04</month>
          <day>11</day>
          <volume>12</volume>
          <issue>4</issue>
          <fpage>e053938</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmjopen.bmj.com/lookup/pmidlookup?view=long&#38;pmid=35410924"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2021-053938</pub-id>
          <pub-id pub-id-type="medline">35410924</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2021-053938</pub-id>
          <pub-id pub-id-type="pmcid">PMC9002256</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Delen</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Eryarsoy</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Davazdahemami</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>No place like home: cross-national data analysis of the efficacy of social distancing during the COVID-19 pandemic</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2020</year>
          <month>05</month>
          <day>28</day>
          <volume>6</volume>
          <issue>2</issue>
          <fpage>e19862</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2020/2/e19862/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/19862</pub-id>
          <pub-id pub-id-type="medline">32434145</pub-id>
          <pub-id pub-id-type="pii">v6i2e19862</pub-id>
          <pub-id pub-id-type="pmcid">PMC7257477</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Emanuel</surname>
              <given-names>EJ</given-names>
            </name>
            <name name-style="western">
              <surname>Persad</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Upshur</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Thome</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Parker</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Glickman</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Boyle</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Smith</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Phillips</surname>
              <given-names>JP</given-names>
            </name>
          </person-group>
          <article-title>Fair allocation of scarce medical resources in the time of Covid-19</article-title>
          <source>N Engl J Med</source>
          <year>2020</year>
          <month>05</month>
          <day>21</day>
          <volume>382</volume>
          <issue>21</issue>
          <fpage>2049</fpage>
          <lpage>2055</lpage>
          <pub-id pub-id-type="doi">10.1056/NEJMsb2005114</pub-id>
          <pub-id pub-id-type="medline">32202722</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="web">
          <article-title>Advice for the public: Coronavirus disease (COVID-19)</article-title>
          <source>World Health Organization</source>
          <access-date>2020-11-18</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.who.int/emergencies/diseases/novel-coronavirus-2019/advice-for-public">https://www.who.int/emergencies/diseases/novel-coronavirus-2019/advice-for-public</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Dredze</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Broniatowski</surname>
              <given-names>DA</given-names>
            </name>
          </person-group>
          <article-title>The Twitter Social Mobility Index: measuring social distancing practices with geolocated tweets</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>12</month>
          <day>03</day>
          <volume>22</volume>
          <issue>12</issue>
          <fpage>e21499</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/12/e21499/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/21499</pub-id>
          <pub-id pub-id-type="medline">33048823</pub-id>
          <pub-id pub-id-type="pii">v22i12e21499</pub-id>
          <pub-id pub-id-type="pmcid">PMC7717895</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lyu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Takikawa</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>The disparity and dynamics of social distancing behaviors in Japan: investigation of mobile phone mobility data</article-title>
          <source>JMIR Med Inform</source>
          <year>2022</year>
          <month>03</month>
          <day>22</day>
          <volume>10</volume>
          <issue>3</issue>
          <fpage>e31557</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2022/3/e31557/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/31557</pub-id>
          <pub-id pub-id-type="medline">35297764</pub-id>
          <pub-id pub-id-type="pii">v10i3e31557</pub-id>
          <pub-id pub-id-type="pmcid">PMC8942095</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Daughton</surname>
              <given-names>AR</given-names>
            </name>
            <name name-style="western">
              <surname>Shelley</surname>
              <given-names>CD</given-names>
            </name>
            <name name-style="western">
              <surname>Barnard</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gerts</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Watson Ross</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Crooker</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Nadiga</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Mukundan</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Vaquera Chavez</surname>
              <given-names>NY</given-names>
            </name>
            <name name-style="western">
              <surname>Parikh</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Pitts</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Fairchild</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Mining and validating social media data for COVID-19-related human behaviors between January and July 2020: infodemiology study</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>05</month>
          <day>25</day>
          <volume>23</volume>
          <issue>5</issue>
          <fpage>e27059</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/5/e27059/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/27059</pub-id>
          <pub-id pub-id-type="medline">33882015</pub-id>
          <pub-id pub-id-type="pii">v23i5e27059</pub-id>
          <pub-id pub-id-type="pmcid">PMC8153035</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hanratty</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bradley</surname>
              <given-names>DT</given-names>
            </name>
            <name name-style="western">
              <surname>Miller</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Dempster</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Determinants of health behaviours intended to prevent spread of respiratory pathogens that have pandemic potential: A rapid review</article-title>
          <source>Acta Psychol (Amst)</source>
          <year>2021</year>
          <month>10</month>
          <volume>220</volume>
          <fpage>103423</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0001-6918(21)00173-6"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.actpsy.2021.103423</pub-id>
          <pub-id pub-id-type="medline">34624664</pub-id>
          <pub-id pub-id-type="pii">S0001-6918(21)00173-6</pub-id>
          <pub-id pub-id-type="pmcid">PMC8492069</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fridman</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Lucas</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Henke</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Zigler</surname>
              <given-names>CK</given-names>
            </name>
          </person-group>
          <article-title>Association between public knowledge about COVID-19, trust in information sources, and adherence to social distancing: cross-sectional survey</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2020</year>
          <month>09</month>
          <day>15</day>
          <volume>6</volume>
          <issue>3</issue>
          <fpage>e22060</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2020/3/e22060/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/22060</pub-id>
          <pub-id pub-id-type="medline">32930670</pub-id>
          <pub-id pub-id-type="pii">v6i3e22060</pub-id>
          <pub-id pub-id-type="pmcid">PMC7511226</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Al-Hasan</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Khuntia</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Yim</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Threat, coping, and social distance adherence during COVID-19: cross-continental comparison using an online cross-sectional survey</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>11</month>
          <day>18</day>
          <volume>22</volume>
          <issue>11</issue>
          <fpage>e23019</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/11/e23019/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/23019</pub-id>
          <pub-id pub-id-type="medline">33119538</pub-id>
          <pub-id pub-id-type="pii">v22i11e23019</pub-id>
          <pub-id pub-id-type="pmcid">PMC7677591</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bowman</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Kwok</surname>
              <given-names>KO</given-names>
            </name>
            <name name-style="western">
              <surname>Redd</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Yi</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Ward</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Wei</surname>
              <given-names>WI</given-names>
            </name>
            <name name-style="western">
              <surname>Atchison</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>SY</given-names>
            </name>
          </person-group>
          <article-title>Comparing public perceptions and preventive behaviors during the early phase of the COVID-19 pandemic in Hong Kong and the United Kingdom: cross-sectional survey study</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>03</month>
          <day>08</day>
          <volume>23</volume>
          <issue>3</issue>
          <fpage>e23231</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/3/e23231/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/23231</pub-id>
          <pub-id pub-id-type="medline">33539309</pub-id>
          <pub-id pub-id-type="pii">v23i3e23231</pub-id>
          <pub-id pub-id-type="pmcid">PMC7942393</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <article-title>Flatten The Curve</article-title>
          <source>Ohio Department of Health</source>
          <year>2020</year>
          <month>04</month>
          <day>09</day>
          <access-date>2021-01-21</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.youtube.com/watch?v=o4PnSYAqQHU">https://www.youtube.com/watch?v=o4PnSYAqQHU</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>An</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Bacon</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hawley</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Russell</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Huffman</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Resnicow</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Relationship between coronavirus-related eHealth literacy and COVID-19 knowledge, attitudes, and practices among US adults: web-based survey study</article-title>
          <source>J Med Internet Res</source>
          <year>2021</year>
          <month>03</month>
          <day>29</day>
          <volume>23</volume>
          <issue>3</issue>
          <fpage>e25042</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2021/3/e25042/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/25042</pub-id>
          <pub-id pub-id-type="medline">33626015</pub-id>
          <pub-id pub-id-type="pii">v23i3e25042</pub-id>
          <pub-id pub-id-type="pmcid">PMC8006897</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hart</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Byrne-Davis</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Epton</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Ghio</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Drury</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Armitage</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Shorter</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Arden</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Chadwick</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Kamal</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Optimising physical distancing to reduce the spread of Covid-19: Behavioural science and disease prevention guidance for public health. Taking a Behavioural Science approach</article-title>
          <source>Queen's University Belfast</source>
          <year>2021</year>
          <month>07</month>
          <day>30</day>
          <access-date>2023-04-14</access-date>
          <publisher-name>British Psychological Society</publisher-name>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pure.qub.ac.uk/en/publications/optimising-physical-distancing-to-reduce-the-spread-of-covid-19-b">https://pure.qub.ac.uk/en/publications/optimising-physical-distancing-to-reduce-the-spread-of-covid-19-b</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hakim</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Bettinger</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Chambers</surname>
              <given-names>CT</given-names>
            </name>
            <name name-style="western">
              <surname>Driedger</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Dubé</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Gavaruzzi</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Giguere</surname>
              <given-names>AMC</given-names>
            </name>
            <name name-style="western">
              <surname>Kavanagh</surname>
              <given-names>É</given-names>
            </name>
            <name name-style="western">
              <surname>Leask</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>MacDonald</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Orji</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Parent</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Paquette</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Roberge</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sander</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Scherer</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Tremblay-Breault</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Reinharz</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Witteman</surname>
              <given-names>HO</given-names>
            </name>
          </person-group>
          <article-title>A web application about herd immunity using personalized avatars: development study</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>10</month>
          <day>30</day>
          <volume>22</volume>
          <issue>10</issue>
          <fpage>e20113</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/10/e20113/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/20113</pub-id>
          <pub-id pub-id-type="medline">33124994</pub-id>
          <pub-id pub-id-type="pii">v22i10e20113</pub-id>
          <pub-id pub-id-type="pmcid">PMC7665952</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hogg</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Customer journey mapping: The path to loyalty</article-title>
          <source>Think with Google</source>
          <year>2018</year>
          <month>02</month>
          <access-date>2021-01-05</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.thinkwithgoogle.com/consumer-insights/consumer-journey/customer-journey-mapping/">https://www.thinkwithgoogle.com/consumer-insights/consumer-journey/customer-journey-mapping/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rising</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Janoff</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>The Scrum software development process for small teams</article-title>
          <source>IEEE Softw</source>
          <year>2000</year>
          <volume>17</volume>
          <issue>4</issue>
          <fpage>26</fpage>
          <lpage>32</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://ieeexplore-ieee-org.acces.bibl.ulaval.ca/document/854065"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/52.854065</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Meso</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Jain</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Agile software development: adaptive systems principles and best practices</article-title>
          <source>Information Systems Management</source>
          <year>2006</year>
          <month>06</month>
          <volume>23</volume>
          <issue>3</issue>
          <fpage>19</fpage>
          <lpage>30</lpage>
          <pub-id pub-id-type="doi">10.1201/1078.10580530/46108.23.3.20060601/93704.3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alsaqqa</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sawalha</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Abdel-Nabi</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Agile software development: methodologies and trends</article-title>
          <source>Int. J. Interact. Mob. Technol</source>
          <year>2020</year>
          <month>07</month>
          <day>10</day>
          <volume>14</volume>
          <issue>11</issue>
          <fpage>246</fpage>
          <pub-id pub-id-type="doi">10.3991/ijim.v14i11.13269</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>DiPierro</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The rise of JavaScript</article-title>
          <source>Comput. Sci. Eng</source>
          <year>2018</year>
          <month>1</month>
          <volume>20</volume>
          <issue>1</issue>
          <fpage>9</fpage>
          <lpage>10</lpage>
          <pub-id pub-id-type="doi">10.1109/MCSE.2018.011111120</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="web">
          <article-title>The Agile Manifesto</article-title>
          <source>Agile Alliance</source>
          <access-date>2021-01-05</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.agilealliance.org/agile101/the-agile-manifesto/">https://www.agilealliance.org/agile101/the-agile-manifesto/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Messager</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <source>Agile Project Management: with Scrum, Lean, Extreme Programming</source>
          <year>2013</year>
          <month>02</month>
          <day>21</day>
          <publisher-loc>Paris, France</publisher-loc>
          <publisher-name>Eyrolles Group</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="web">
          <article-title>What is Agile?</article-title>
          <source>Agile Alliance</source>
          <access-date>2020-12-18</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.agilealliance.org/agile101/">https://www.agilealliance.org/agile101/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Cook</surname>
              <given-names>AR</given-names>
            </name>
            <name name-style="western">
              <surname>Lim</surname>
              <given-names>JT</given-names>
            </name>
            <name name-style="western">
              <surname>Sun</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Dickens</surname>
              <given-names>BL</given-names>
            </name>
          </person-group>
          <article-title>A systematic review of COVID-19 epidemiology based on current evidence</article-title>
          <source>J Clin Med</source>
          <year>2020</year>
          <month>03</month>
          <day>31</day>
          <volume>9</volume>
          <issue>4</issue>
          <fpage>1</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=jcm9040967"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/jcm9040967</pub-id>
          <pub-id pub-id-type="medline">32244365</pub-id>
          <pub-id pub-id-type="pii">jcm9040967</pub-id>
          <pub-id pub-id-type="pmcid">PMC7231098</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="web">
          <article-title>COVID-19: Prevention and risks</article-title>
          <source>Public Health Agency of Canada</source>
          <access-date>2022-03-30</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.canada.ca/en/public-health/services/diseases/2019-novel-coronavirus-infection/prevention-risks.html">https://www.canada.ca/en/public-health/services/diseases/2019-novel-coronavirus-infection/prevention-risks.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="web">
          <article-title>Google analytics cookie usage on websites</article-title>
          <source>Google Developers</source>
          <access-date>2022-01-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://developers.google.com/analytics/devguides/collection/analyticsjs/cookie-usage">https://developers.google.com/analytics/devguides/collection/analyticsjs/cookie-usage</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="web">
          <article-title>How users are identified for user metrics</article-title>
          <source>Google Help Center</source>
          <access-date>2020-09-08</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://support.google.com/analytics/answer/2992042?hl=en">https://support.google.com/analytics/answer/2992042?hl=en</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="web">
          <article-title>ISO 639 language codes</article-title>
          <source>ISO</source>
          <access-date>2021-03-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.iso.org/iso-639-language-codes.html?hl=en_US">https://www.iso.org/iso-639-language-codes.html?hl=en_US</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="web">
          <article-title>ISO 3166 Country Codes</article-title>
          <source>ISO</source>
          <access-date>2021-03-26</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.iso.org/iso-3166-country-codes.html?hl=en_US">https://www.iso.org/iso-3166-country-codes.html?hl=en_US</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lenth</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>emmeans: Estimated Marginal Means, aka Least-Squares Means</article-title>
          <source>R Foundation for Statistical Computing</source>
          <access-date>2022-01-22</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://CRAN.R-project.org/package=emmeans">https://CRAN.R-project.org/package=emmeans</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <collab>R Core Team</collab>
          </person-group>
          <article-title>R: A language and environment for statistical computing</article-title>
          <source>R Foundation for Statistical Computing</source>
          <year>2019</year>
          <access-date>2023-04-14</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.R-project.org/">https://www.R-project.org/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gough</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hunter</surname>
              <given-names>RF</given-names>
            </name>
            <name name-style="western">
              <surname>Ajao</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Jurek</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>McKeown</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Hong</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Barrett</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Ferguson</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>McElwee</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>McCarthy</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kee</surname>
              <given-names>F</given-names>
            </name>
          </person-group>
          <article-title>Tweet for behavior change: using social media for the dissemination of public health messages</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2017</year>
          <month>03</month>
          <day>23</day>
          <volume>3</volume>
          <issue>1</issue>
          <fpage>e14</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2017/1/e14/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/publichealth.6313</pub-id>
          <pub-id pub-id-type="medline">28336503</pub-id>
          <pub-id pub-id-type="pii">v3i1e14</pub-id>
          <pub-id pub-id-type="pmcid">PMC5383801</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tannenbaum</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Greaves</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Graham</surname>
              <given-names>ID</given-names>
            </name>
          </person-group>
          <article-title>Why sex and gender matter in implementation research</article-title>
          <source>BMC Med Res Methodol</source>
          <year>2016</year>
          <month>10</month>
          <day>27</day>
          <volume>16</volume>
          <issue>1</issue>
          <fpage>145</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedresmethodol.biomedcentral.com/articles/10.1186/s12874-016-0247-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12874-016-0247-7</pub-id>
          <pub-id pub-id-type="medline">27788671</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12874-016-0247-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC5084413</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Doogan</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Buntine</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Linger</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Brunt</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Public perceptions and attitudes toward COVID-19 nonpharmaceutical interventions across six countries: a topic modeling analysis of Twitter data</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>09</month>
          <day>03</day>
          <volume>22</volume>
          <issue>9</issue>
          <fpage>e21419</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/9/e21419/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/21419</pub-id>
          <pub-id pub-id-type="medline">32784190</pub-id>
          <pub-id pub-id-type="pii">v22i9e21419</pub-id>
          <pub-id pub-id-type="pmcid">PMC7505256</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mohamad</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Tham</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Ayub</surname>
              <given-names>SH</given-names>
            </name>
            <name name-style="western">
              <surname>Hamzah</surname>
              <given-names>MR</given-names>
            </name>
            <name name-style="western">
              <surname>Hashim</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Azlan</surname>
              <given-names>AA</given-names>
            </name>
          </person-group>
          <article-title>Relationship between COVID-19 information sources and attitudes in battling the pandemic among the Malaysian public: cross-sectional survey study</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>11</month>
          <day>12</day>
          <volume>22</volume>
          <issue>11</issue>
          <fpage>e23922</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/11/e23922/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/23922</pub-id>
          <pub-id pub-id-type="medline">33151897</pub-id>
          <pub-id pub-id-type="pii">v22i11e23922</pub-id>
          <pub-id pub-id-type="pmcid">PMC7674144</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Boon-Itt</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Skunkan</surname>
              <given-names>Y</given-names>
            </name>
          </person-group>
          <article-title>Public perception of the COVID-19 pandemic on Twitter: sentiment analysis and topic modeling study</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2020</year>
          <month>11</month>
          <day>11</day>
          <volume>6</volume>
          <issue>4</issue>
          <fpage>e21978</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2020/4/e21978/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/21978</pub-id>
          <pub-id pub-id-type="medline">33108310</pub-id>
          <pub-id pub-id-type="pii">v6i4e21978</pub-id>
          <pub-id pub-id-type="pmcid">PMC7661106</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gouin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>MacNeil</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Switzer</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Carrese-Chacra</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Durif</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Knäuper</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Socio-demographic, social, cognitive, and emotional correlates of adherence to physical distancing during the COVID-19 pandemic: a cross-sectional study</article-title>
          <source>Can J Public Health</source>
          <year>2021</year>
          <month>03</month>
          <volume>112</volume>
          <issue>1</issue>
          <fpage>17</fpage>
          <lpage>28</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/33464556"/>
          </comment>
          <pub-id pub-id-type="doi">10.17269/s41997-020-00457-5</pub-id>
          <pub-id pub-id-type="medline">33464556</pub-id>
          <pub-id pub-id-type="pii">10.17269/s41997-020-00457-5</pub-id>
          <pub-id pub-id-type="pmcid">PMC7814525</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alshareef</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Yunusa</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Al-Hanawi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>The influence of COVID-19 information sources on the attitudes and practices toward COVID-19 among the general public of Saudi Arabia: cross-sectional online survey study</article-title>
          <source>JMIR Public Health Surveill</source>
          <year>2021</year>
          <month>07</month>
          <day>30</day>
          <volume>7</volume>
          <issue>7</issue>
          <fpage>e28888</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://publichealth.jmir.org/2021/7/e28888/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/28888</pub-id>
          <pub-id pub-id-type="medline">34081610</pub-id>
          <pub-id pub-id-type="pii">v7i7e28888</pub-id>
          <pub-id pub-id-type="pmcid">PMC8330635</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="web">
          <article-title>COVID-19 Strategy update</article-title>
          <source>World Health Organization</source>
          <year>2020</year>
          <month>04</month>
          <day>14</day>
          <access-date>2023-04-14</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.who.int/publications/m/item/covid-19-strategy-update">https://www.who.int/publications/m/item/covid-19-strategy-update</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hakim</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Provencher</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Chambers</surname>
              <given-names>CT</given-names>
            </name>
            <name name-style="western">
              <surname>Driedger</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Dube</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Gavaruzzi</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Giguere</surname>
              <given-names>AMC</given-names>
            </name>
            <name name-style="western">
              <surname>Ivers</surname>
              <given-names>NM</given-names>
            </name>
            <name name-style="western">
              <surname>MacDonald</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Paquette</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Reinharz</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Witteman</surname>
              <given-names>HO</given-names>
            </name>
          </person-group>
          <article-title>Interventions to help people understand community immunity: A systematic review</article-title>
          <source>Vaccine</source>
          <year>2019</year>
          <month>01</month>
          <day>07</day>
          <volume>37</volume>
          <issue>2</issue>
          <fpage>235</fpage>
          <lpage>247</lpage>
          <pub-id pub-id-type="doi">10.1016/j.vaccine.2018.11.016</pub-id>
          <pub-id pub-id-type="medline">30528593</pub-id>
          <pub-id pub-id-type="pii">S0264-410X(18)31529-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hekler</surname>
              <given-names>EB</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Collins</surname>
              <given-names>LM</given-names>
            </name>
            <name name-style="western">
              <surname>Doherty</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hollis</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Rivera</surname>
              <given-names>DE</given-names>
            </name>
            <name name-style="western">
              <surname>West</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Wyatt</surname>
              <given-names>JC</given-names>
            </name>
          </person-group>
          <article-title>Evaluating digital health interventions: key questions and approaches</article-title>
          <source>Am J Prev Med</source>
          <year>2016</year>
          <month>11</month>
          <volume>51</volume>
          <issue>5</issue>
          <fpage>843</fpage>
          <lpage>851</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://europepmc.org/abstract/MED/27745684"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.amepre.2016.06.008</pub-id>
          <pub-id pub-id-type="medline">27745684</pub-id>
          <pub-id pub-id-type="pii">S0749-3797(16)30229-X</pub-id>
          <pub-id pub-id-type="pmcid">PMC5324832</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
