<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JFR</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Form Res</journal-id>
      <journal-title>JMIR Formative Research</journal-title>
      <issn pub-type="epub">2561-326X</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v3i3e11617</article-id>
      <article-id pub-id-type="pmid">31573909</article-id>
      <article-id pub-id-type="doi">10.2196/11617</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>A Mobile Patient-Reported Outcome Measure App With Talking Touchscreen: Usability Assessment</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Stängle</surname>
            <given-names>Sabrina</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Jaspers</surname>
            <given-names>Monique</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Latulippe</surname>
            <given-names>Karine</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Welbie</surname>
            <given-names>Marlies</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Research Group Lifestyle and Health</institution>
            <institution>Research Center Healthy and Sustainable Living</institution>
            <institution>Utrecht University of Applied Sciences</institution>
            <addr-line>Postbus 12011</addr-line>
            <addr-line>Utrecht, 3501 AA</addr-line>
            <country>Netherlands</country>
            <phone>31 638192100</phone>
            <email>marlies.welbie@hu.nl</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1697-1357</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Wittink</surname>
            <given-names>Harriet</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-5592-2879</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Westerman</surname>
            <given-names>Marjan J</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9827-6958</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Topper</surname>
            <given-names>Ilse</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-5295-7707</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Snoei</surname>
            <given-names>Josca</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-3489-587X</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author">
          <name name-style="western">
            <surname>Devillé</surname>
            <given-names>Walter L J M</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <xref rid="aff4" ref-type="aff">4</xref>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9854-1718</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Research Group Lifestyle and Health</institution>
        <institution>Research Center Healthy and Sustainable Living</institution>
        <institution>Utrecht University of Applied Sciences</institution>
        <addr-line>Utrecht</addr-line>
        <country>Netherlands</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Institute of Health Sciences, Amsterdam Public Health Research Institute</institution>
        <institution>Department of Methodology and Statistics</institution>
        <institution>VU University</institution>
        <addr-line>Amsterdam</addr-line>
        <country>Netherlands</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Julius Centre for Health Sciences and Primary Care</institution>
        <institution>University Medical Centre Utrecht</institution>
        <addr-line>Utrecht</addr-line>
        <country>Netherlands</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Faculty of Social and Behavioral Sciences</institution>
        <institution>University of Amsterdam</institution>
        <addr-line>Amsterdam</addr-line>
        <country>Netherlands</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>Dutch Centre of Expertise on Health Disparities (Pharos)</institution>
        <addr-line>Utrecht</addr-line>
        <country>Netherlands</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Marlies Welbie <email>marlies.welbie@hu.nl</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <season>Jul-Sep</season>
        <year>2019</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>27</day>
        <month>9</month>
        <year>2019</year>
      </pub-date>
      <volume>3</volume>
      <issue>3</issue>
      <elocation-id>e11617</elocation-id>
      <history>
        <date date-type="received">
          <day>18</day>
          <month>7</month>
          <year>2018</year>
        </date>
        <date date-type="rev-request">
          <day>8</day>
          <month>10</month>
          <year>2018</year>
        </date>
        <date date-type="rev-recd">
          <day>1</day>
          <month>3</month>
          <year>2019</year>
        </date>
        <date date-type="accepted">
          <day>24</day>
          <month>3</month>
          <year>2019</year>
        </date>
      </history>
      <copyright-statement>©Marlies Welbie, Harriet Wittink, Marjan J Westerman, Ilse Topper, Josca Snoei, Walter L J M Devillé. Originally published in JMIR Formative Research (http://formative.jmir.org), 27.09.2019</copyright-statement>
      <copyright-year>2019</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Formative Research, is properly cited. The complete bibliographic information, a link to the original publication on http://formative.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://formative.jmir.org/2019/3/e11617" xlink:type="simple"/>
      <abstract>
        <sec sec-type="Background">
          <title>Background</title>
          <p>In the past years, a mobile health (mHealth) app called the Dutch Talking Touch Screen Questionnaire (DTTSQ) was developed in The Netherlands. The aim of development was to enable Dutch physical therapy patients to autonomously complete a health-related questionnaire regardless of their level of literacy and digital skills.</p>
        </sec>
        <sec sec-type="Objective">
          <title>Objective</title>
          <p>The aim of this study was to evaluate the usability (defined as the effectiveness, efficiency, and satisfaction) of the prototype of the DTTSQ for Dutch physical therapy patients with diverse levels of experience in using mobile technology.</p>
        </sec>
        <sec sec-type="Methods">
          <title>Methods</title>
          <p>The qualitative Three-Step Test-Interview method, including both think-aloud and retrospective probing techniques, was used to gain insight into the usability of the DTTSQ. A total of 24 physical therapy patients were included. The interview data were analyzed using a thematic content analysis approach aimed at analyzing the accuracy and completeness with which participants completed the questionnaire (effectiveness), the time it took the participants to complete the questionnaire (efficiency), and the extent to which the participants were satisfied with the ease of use of the questionnaire (satisfaction). The problems encountered by the participants in this study were given a severity rating that was used to provide a rough estimate of the need for additional usability efforts.</p>
        </sec>
        <sec sec-type="Results">
          <title>Results</title>
          <p>All participants within this study were very satisfied with the ease of use of the DTTSQ. Overall, 9 participants stated that the usability of the app exceeded their expectations. The group of 4 average-/high-experienced participants encountered only 1 problem in total, whereas the 11 little-experienced participants encountered an average of 2 problems per person and the 9 inexperienced participants an average of 3 problems per person. A total of 13 different kind of problems were found during this study. Of these problems, 4 need to be addressed before the DTTSQ will be released because they have the potential to negatively influence future usage of the tool. The other 9 problems were less likely to influence future usage of the tool substantially.</p>
        </sec>
        <sec sec-type="Conclusions">
          <title>Conclusions</title>
          <p>The usability of the DTTSQ needs to be improved before it can be released. No problems were found with satisfaction or efficiency during the usability test. The effectiveness needs to be improved by (1) making it easier to navigate through screens without the possibility of accidentally skipping one, (2) enabling the possibility to insert an answer by tapping on the text underneath a photograph instead of just touching the photograph itself, and (3) making it easier to correct wrong answers. This study shows the importance of including less skilled participants in a usability study when striving for inclusive design and the importance of measuring not just satisfaction but also efficiency and effectiveness during such studies.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>mHealth</kwd>
        <kwd> eHealth</kwd>
        <kwd> surveys and questionnaires</kwd>
        <kwd> physical therapy specialty</kwd>
        <kwd> qualitative research</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Digital Divide</title>
        <p>Electronic health (eHealth) is developing rapidly [<xref ref-type="bibr" rid="ref1">1</xref>]. It is defined as the use of information and communication technology (ICT) in health care [<xref ref-type="bibr" rid="ref2">2</xref>]. A growing amount of literature indicates that using eHealth can improve the accessibility, quality, and efficiency of health care [<xref ref-type="bibr" rid="ref3">3</xref>-<xref ref-type="bibr" rid="ref5">5</xref>]. It seems to be effective for people who have access to it and are able to use it well, which is not the case for everybody [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>]. For instance, people with low income or low education and people who are 65 years and older are vulnerable when it comes to effective eHealth use. In these populations, access to the internet and hardware, such as personal computers, tablets, mobile phones, and smartphones, and the experience and skills to use these devices is low [<xref ref-type="bibr" rid="ref6">6</xref>-<xref ref-type="bibr" rid="ref9">9</xref>]. Differences between people regarding digital skills and access to the internet and hardware is often referred to as the digital divide [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. As eHealth technologies are usually primarily developed for people who are experienced and skilled in using ICT [<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref13">13</xref>], people who do not have access to ICT or are not skilled in using it are at risk of being excluded from the use of eHealth. Looking at the widespread expansion of eHealth technologies, this encompasses the potential threat of contributing to the ongoing exacerbation of health inequalities in Western countries [<xref ref-type="bibr" rid="ref1">1</xref>]. However, if the needs, preferences, capacities, values, and goals of potential users who do not have good access to the internet and digital technology or who are not well skilled in using this technology would be explored and taken into account during each stage of development of eHealth tools, eHealth could potentially <italic>reduce</italic> health inequalities [<xref ref-type="bibr" rid="ref14">14</xref>].</p>
      </sec>
      <sec>
        <title>Mobile Technology Reduces Digital Divide</title>
        <p>The development of a specific form of eHealth technology, called mobile health (mHealth) technology, seems especially promising when it comes to reducing health inequalities [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref15">15</xref>-<xref ref-type="bibr" rid="ref17">17</xref>]. mHealth has been defined by the Global Observatory for eHealth of the World Health Organization as “medical and public health practice supported by mobile devices, such as mobile phones, patient monitoring devices, personal digital assistants, and other wireless devices” [<xref ref-type="bibr" rid="ref18">18</xref>]. A recent project called eSalud showed that mHealth can be cost-effective, help to overcome cultural and language barriers, and provide health information and services to low–health access areas [<xref ref-type="bibr" rid="ref15">15</xref>]. Furthermore, recent publications indicate that the digital divide is narrowing because of the increased ownership of mobile devices such as smartphones and tablets [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref17">17</xref>].</p>
      </sec>
      <sec>
        <title>Inclusive Mobile Health Design Could Potentially Reduce Health Inequalities</title>
        <p>Still, having access to the internet and digital technology does not automatically mean that people are able and willing to use it effectively to increase their health or that different people use it in the same way [<xref ref-type="bibr" rid="ref14">14</xref>,<xref ref-type="bibr" rid="ref19">19</xref>-<xref ref-type="bibr" rid="ref25">25</xref>]. Recent studies found ethnic and socioeconomic differences in mHealth usage [<xref ref-type="bibr" rid="ref19">19</xref>,<xref ref-type="bibr" rid="ref20">20</xref>], and it is known that older people use mHealth differently from younger people [<xref ref-type="bibr" rid="ref14">14</xref>]. In addition, though the gap of people owning tablets and smartphones between groups is closing, still a substantial number of people do not own such devices. For instance, the percentage of Dutch citizens of 65 years and older owning a tablet computer in 2017 was 55.2% versus 75.8% citizens of 12 to 25 years of age [<xref ref-type="bibr" rid="ref26">26</xref>]. Considering that vulnerable groups, such as people with low income and low education, bear a disproportionate burden of disease [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>] and the number of health care visits increases with age [<xref ref-type="bibr" rid="ref29">29</xref>], it is to be expected that a relatively large number of care recipients do not have a lot of experience in using mobile technology. To fulfill the promise of mHealth technology contributing to a reduction of health inequalities, it is very important to carefully test the usability of mHealth apps in research populations, which include members of the target populations that are at risk of being excluded from usage of the tested tool.</p>
      </sec>
      <sec>
        <title>Development of the Dutch Talking Touch Screen Questionnaire</title>
        <p>In the past years, a prototype of an mHealth app, called the Dutch Talking Touch Screen Questionnaire (DTTSQ), was developed in The Netherlands. The idea of developing a talking touch screen was inspired by the work of Hahn and Cella [<xref ref-type="bibr" rid="ref30">30</xref>]. The aim of developing the DTTSQ was to enable Dutch physical therapy patients to autonomously complete a user-friendly health-related questionnaire regardless of their literacy and digital skills. As it is not to be expected that all physical therapy patients own a tablet computer, the DTTSQ is meant to be presented in a physical therapy practice on a tablet computer that is owned by the physical therapy practice concerned. Patients are asked to complete the DTTSQ in the waiting room of the physical therapist before their first visit. The development of the prototype of the DTTSQ, which runs on a tablet computer, was described in detail by Cremers et al in 2015 [<xref ref-type="bibr" rid="ref31">31</xref>]. Before this study, the prototype was only tested in a sample <italic>outside</italic> of the physical therapy context.</p>
        <p>The aim of this study was to test the prototype of the DTTSQ within the physical therapy context to see what parts of the prototype needed adjustment for it to be user-friendly for physical therapy patients regardless of their level of experience with operating mobile technology.</p>
        <p>The research question underlying this study was:</p>
        <disp-quote>
          <p>
        What is the usability of the prototype of the DTTSQ for physical therapy patients with different levels of experience in using mobile technology?</p>
        </disp-quote>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Design</title>
        <p>A qualitative descriptive study was carried out. Observational data on the way participants operated the DTTSQ were collected through the Three-Step Test-Interview (TSTI) method [<xref ref-type="bibr" rid="ref32">32</xref>]. This method includes both think-aloud and retrospective probing techniques.</p>
      </sec>
      <sec>
        <title>Definitions</title>
        <p><italic>Usability</italic> was defined by the International Standards Organization as “the effectiveness, efficiency and satisfaction with which specified users can achieve goals in particular environments” [<xref ref-type="bibr" rid="ref33">33</xref>]. <italic>Effectiveness</italic> is the accuracy and completeness with which users achieve certain goals [<xref ref-type="bibr" rid="ref34">34</xref>]. In this study, problem rates and severity of problems were used as the primary indicator of effectiveness. <italic>Efficiency</italic> is the relation between the accuracy and completeness with which users achieve certain goals and the resources expended in achieving them [<xref ref-type="bibr" rid="ref34">34</xref>]. In this study, completion time was used as an indicator of efficiency. <italic>Satisfaction</italic> is the users’ comfort with and positive attitudes toward the use of a system [<xref ref-type="bibr" rid="ref34">34</xref>]. In this study, participants were interviewed about their satisfaction with the ease of use of the DTTSQ. Ease of use was defined as the degree to which the usage of a particular system is free from effort [<xref ref-type="bibr" rid="ref35">35</xref>].</p>
      </sec>
      <sec>
        <title>Setting and Participant Selection</title>
        <p>Data were collected in the same study population and at the same time as the data reported in a paper earlier published by Welbie et al [<xref ref-type="bibr" rid="ref36">36</xref>]. Recruitment took place in 11 primary care practices in deprived areas of Utrecht, The Netherlands. Patients were invited by their physical therapists to participate in this study. The physical therapists shortly explained the goal of the study and provided the patients with an information letter that was written in plain Dutch language. If patients were interested, the physical therapist asked permission to give the patients’ telephone number to researcher IT. Then researcher IT (1) contacted the patient by telephone, (2) again shortly explained the aim of the study, (3) made sure the patient understood what was asked of him/her, (4) answered any question the potential participant may have had, and (5) checked the inclusion criteria. The inclusion criteria for participants were as follows: aged 18 years or older, Dutch as their first language, and the patients and both their parents were born in The Netherlands. This last inclusion criterion was added because in a following study, the usability of a direct Turkish translation of the DTTSQ will be tested. For the outcomes of both studies to be comparable, it is important that the cultural background of participants of this study was not <italic>mixed</italic>. This last inclusion criterion excludes second-generation immigrants with a non-Dutch background. The sampling procedure was aimed at getting a broad variation in levels of education and age plus balance in our sample regarding gender. Age was used as a proxy for level of experience with using mobile technology because with increase in age, the experience with mobile devices decreases [<xref ref-type="bibr" rid="ref26">26</xref>]. Taking age as a selection criterion was more practical for the recruiting physical therapists, as this is noted standardly in patient files. By making sure that there was variation in age, it was expected to find variation in experience with mobile devices in the study sample. Throughout the recruitment process, the recruiting physical therapists were constantly kept informed about the profiles of participants the researchers were looking for. In total, 24 physical therapy patients were included in this study [<xref ref-type="bibr" rid="ref36">36</xref>]. Characteristics of the study population can be found in <xref ref-type="table" rid="table1">Tables 1</xref> and <xref ref-type="table" rid="table2">2</xref>.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Characteristics of study population (N=24).</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="570"/>
            <col width="400"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Characteristics</td>
                <td>Study population</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="2">Age (years), mean (range)</td>
                <td>56 (18-79)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Gender, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Male</td>
                <td>9 (38)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Female</td>
                <td>15 (62)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Level of education, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Low<sup>a</sup></td>
                <td>6 (25)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Moderate<sup>b</sup></td>
                <td>13 (54)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>High<sup>c</sup></td>
                <td>5 (21)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Self-declared experience with using mobile technology, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>None</td>
                <td>9 (37)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Little</td>
                <td>11 (46)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Average/high</td>
                <td>4 (17)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>Low: no or at most primary education finished.</p>
            </fn>
            <fn id="table1fn2">
              <p><sup>b</sup>Moderate: lower secondary education, (upper) secondary education, or postsecondary nontertiary education (including vocational education).</p>
            </fn>
            <fn id="table1fn3">
              <p><sup>c</sup>High: tertiary education (bachelor’s degree or higher).</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Characteristics per participant.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="180"/>
            <col width="430"/>
            <col width="150"/>
            <col width="240"/>
            <thead>
              <tr valign="top">
                <td>Pseudonym</td>
                <td>Experience with mobile technology</td>
                <td>Age (years)</td>
                <td>Level of education</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Ida</td>
                <td>None</td>
                <td>66</td>
                <td>Low<sup>a</sup></td>
              </tr>
              <tr valign="top">
                <td>Bill</td>
                <td>None</td>
                <td>72</td>
                <td>Moderate<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td>Mia</td>
                <td>None</td>
                <td>73</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Dora</td>
                <td>None</td>
                <td>77</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>Ilene</td>
                <td>None</td>
                <td>79</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>Bob</td>
                <td>None</td>
                <td>68</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Jerome</td>
                <td>None</td>
                <td>47</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>Helga</td>
                <td>None</td>
                <td>54</td>
                <td>High<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>Michelle</td>
                <td>None</td>
                <td>56</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>Roger</td>
                <td>Little</td>
                <td>70</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Peter</td>
                <td>Little</td>
                <td>18</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Christine</td>
                <td>Little</td>
                <td>39</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Jill</td>
                <td>Little</td>
                <td>55</td>
                <td>High</td>
              </tr>
              <tr valign="top">
                <td>Lydia</td>
                <td>Little</td>
                <td>56</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Rose</td>
                <td>Little</td>
                <td>60</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Francine</td>
                <td>Little</td>
                <td>61</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Harald</td>
                <td>Little</td>
                <td>63</td>
                <td>High</td>
              </tr>
              <tr valign="top">
                <td>Henry</td>
                <td>Little</td>
                <td>64</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Ronald</td>
                <td>Little</td>
                <td>70</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>Bernie</td>
                <td>Little</td>
                <td>76</td>
                <td>High</td>
              </tr>
              <tr valign="top">
                <td>Jude</td>
                <td>Average/high</td>
                <td>18</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Joline</td>
                <td>Average/high</td>
                <td>19</td>
                <td>Moderate</td>
              </tr>
              <tr valign="top">
                <td>Ellen</td>
                <td>Average/high</td>
                <td>32</td>
                <td>High</td>
              </tr>
              <tr valign="top">
                <td>Sandra</td>
                <td>Average/high</td>
                <td>39</td>
                <td>Moderate</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table2fn1">
              <p><sup>a</sup>Low: no or at most primary education finished.</p>
            </fn>
            <fn id="table2fn2">
              <p><sup>b</sup>Moderate: lower secondary education, (upper) secondary education, or postsecondary nontertiary education (including vocational education).</p>
            </fn>
            <fn id="table3fn3">
              <p><sup>c</sup>High: tertiary education (bachelor’s degree or higher).</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Content of the Dutch Talking Touch Screen Questionnaire</title>
        <p>The prototype of the DTTSQ was a digital app on a tablet computer. It was developed during a co-design process [<xref ref-type="bibr" rid="ref37">37</xref>], which in this case meant that a group of 10 low-literate people helped to design the questionnaire. As a result of the co-design process, questions on pain location and pain intensity were added to the original questions of an existing questionnaire which aims to select limitations in functioning and to formulate specific treatment goals [<xref ref-type="bibr" rid="ref38">38</xref>,<xref ref-type="bibr" rid="ref39">39</xref>]. Furthermore, visual (videos and photos) and auditory (speech technology) support were added to enable participants to see and hear the questions that were shown on separate screens. Response items could be selected by tapping on the touch screen and plain language was used in all spoken and written text within the DTTSQ [<xref ref-type="bibr" rid="ref31">31</xref>]. An overview of all types of screens is given in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>. The 8 questions of the questionnaire can be found in screenshots 2, 3, 4, 7, 9, 11, 12, and 13, which can be found in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>.</p>
        <sec>
          <title>Instructions</title>
          <p>Instructions were given in the form of 3 video clips:</p>
          <list list-type="order">
            <list-item>
              <p>An introduction clip in which the purpose of the questionnaire and all functions of the questionnaire were explained (see <xref rid="figure1" ref-type="fig">Figure 1</xref> and screenshot 1 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
            </list-item>
            <list-item>
              <p>An instruction clip in which the purpose of question 4 and a newly added navigation function were explained (see screenshot 6 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
            </list-item>
            <list-item>
              <p>A closing clip in which the participant is thanked, explained what the physical therapist would do next, and told that the questionnaire would close down automatically (see screenshot 16 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
            </list-item>
          </list>
          <fig id="figure1" position="float">
            <caption>
              <p>Introduction movie.</p>
            </caption>
            <graphic xlink:href="formative_v3i3e11617_fig1.png" alt-version="no" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Functions</title>
        </sec>
        <sec>
          <title>Next Button</title>
          <p>It is a navigation function to go to the next screen. It is not activated unless a response item is selected (except for question 4; see <xref rid="figure2" ref-type="fig">Figure 2</xref> and screenshot 7 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
        </sec>
        <sec>
          <title>Help Button</title>
          <p>It activates the help function: the text on the screen is read aloud, the purpose of the question is explained, and operating instructions for the particular screen are provided.</p>
        </sec>
        <sec>
          <title>Correction Function</title>
          <p>Tapping a second time on a response item, deselects the item.</p>
        </sec>
        <sec>
          <title>Stop Button</title>
          <p>It is an escape function: it shuts down the questionnaire. All previous given answers are saved.</p>
        </sec>
        <sec>
          <title>Overviews</title>
          <p>To help participants keep track of their answers, overviews of previous given answers were provided regularly during completion of the questionnaire (see <xref rid="figure3" ref-type="fig">Figure 3</xref> and screenshot 5, 8, 10, 14 and 15 in <xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref> screenshots).</p>
          <fig id="figure2" position="float">
            <caption>
              <p>Question 4: “Select the activities in which you are limited”.</p>
            </caption>
            <graphic xlink:href="formative_v3i3e11617_fig2.png" alt-version="no" position="float" xlink:type="simple"/>
          </fig>
          <fig id="figure3" position="float">
            <caption>
              <p>Overview answers total questionnaire:
“On the screen you see an overview of all your answers you provided until now.”.</p>
            </caption>
            <graphic xlink:href="formative_v3i3e11617_fig3.png" alt-version="no" position="float" xlink:type="simple"/>
          </fig>
        </sec>
      </sec>
      <sec>
        <title>Data Collection and Procedures</title>
        <p>Data collection took place at the physical therapy practice or the participant’s home, depending on the preference of the participant. Researchers IT and JS were present. Researcher IT was in the lead during the interviews. Researcher JS asked complementary questions if she missed information.</p>
        <p>The following steps were taken according to the TSTI method [<xref ref-type="bibr" rid="ref32">32</xref>].</p>
        <sec>
          <title>Step 1</title>
          <p>Each participant was observed by researchers IT and JS while they were completing the DTTSQ thinking out loud. This step was aimed at collecting observational data regarding the usability of the DTTSQ. The data collected consisted of 2 kinds: (1) observations of participant’s behavior and (2) think-aloud data. The data were recorded on videotapes as well as audiotapes. In addition, the researchers took real-time notes for use during the following steps of the interviews as well as for later analysis. The researchers wrote their notes down on hardcopies of print screens of the DTTSQ. Researchers IT and JS noted problems with operating the tablet computer, including using the touch screen, navigating through the questionnaire, understanding the task given in each screen, selecting response items, and using the correction function. They also wrote down when the stop button was used. The researchers did not interfere in the completion process by asking any questions or providing help.</p>
        </sec>
        <sec>
          <title>Step 2</title>
          <p>Researcher IT conducted an in-depth interview after the participant finished completing the DTTSQ. Data collection during this step was exclusively focused on filling possible gaps and checking the observational data collected during step 1.</p>
        </sec>
        <sec>
          <title>Step 3</title>
          <p>During step 3 of the TSTI, researcher IT conducted a semistructured interview aimed at eliciting experiences and opinions of the participant. During the interview, each screen was operated in the same way the participant did during step 1 and the same answers were entered. This was done to help the participant to clearly remember all his thoughts and actions during the completion of the questionnaire. Participants were stimulated to report feelings and express opinions, preferences, and recommendations. If they encountered problems in operating the DTTSQ, they were asked what they thought the exact nature and possible cause of each type of problem was and how they tried to overcome the problem. Then, the participants were questioned about their satisfaction regarding the ease of use of the user interface, technical operation, layout and content, and overall usability of the DTTSQ. Researcher JS was allowed to ask complementary questions, if she felt it was necessary, to get complete and enriched data. Researcher IT finished the interview by collecting demographic data and data on self-reported experience with mobile technology (see <xref ref-type="table" rid="table1">Tables 1</xref> and <xref ref-type="table" rid="table2">2</xref>).</p>
        </sec>
      </sec>
      <sec>
        <title>Analyses</title>
        <p>Data were analyzed using a thematic content analysis approach [<xref ref-type="bibr" rid="ref40">40</xref>]. Overall, 4 types of data were analyzed: (1) video recordings of the completion of the questionnaire, (2) field notes of the observed participant behavior, (3) transcriptions of the audio recordings of the semistructured interviews, and (4) background information regarding the educational level, age, gender, and self-reported experience with using mobile technology.</p>
        <p>To get more familiar with the data and to create an overview, researcher MW made a descriptive summary of each case on the basis of all 4 types of generated data. Each summary contained information on whether or not the questionnaire was fully completed, if, when, and why the stop function was used, the kind of problems that occurred with the operation, the completion time, and all emerging themes regarding satisfaction or dissatisfaction with the ease of use of the questionnaire. The summaries were supplemented with information regarding educational level, age, gender, and experience in using mobile technology.</p>
        <p>Subsequently, researcher MW derived the observed problems from the summaries. She clustered the problems. For every new problem, a new category was made. MW analyzed the video recordings to see how many times each problem was made in total, per participant and per question/screen of the questionnaire. After a full overview of problems had emerged, she scored the level of severity of each problem, as described by Nielsen and Loranger [<xref ref-type="bibr" rid="ref41">41</xref>]: low, medium, serious, or critical. To score severity, she used the method of Hattink et al [<xref ref-type="bibr" rid="ref42">42</xref>]. The severity was scored by answering the 3 questions of Nielsen and Loranger [<xref ref-type="bibr" rid="ref41">41</xref>] with <italic>yes</italic> (=one point) or <italic>no</italic> (=0 points):</p>
        <list list-type="order">
          <list-item>
            <p>Frequency: Do a substantial number of users encounter the problem? Within this study, this question was answered with yes if one-third or more participants had encountered the problem.</p>
          </list-item>
          <list-item>
            <p>Impact: Does the problem cause much trouble to those users who encounter it? Within this study, this question was answered with yes if the problem had led at least one participant to stop completing the questionnaire.</p>
          </list-item>
          <list-item>
            <p>Persistence: Does the problem cause trouble repeatedly? Within this study, this question was answered with yes if the problem had occurred with an average of more than one time per participant.</p>
          </list-item>
        </list>
        <p>This resulted in a 0- to 3-point score per problem. Each score was related to a level of severity: 0=low, 1=medium, 2=serious, and 3=critical.</p>
        <p>These severity ratings give an indication of which problems lead to disastrous usability problems and which problems are more cosmetic in nature [<xref ref-type="bibr" rid="ref43">43</xref>]. This provides insight into whether or not the usability of the DTTSQ needs to be improved before it can be released. Nielsen and Loranger recommend tackling only serious and critical severe problems during the development process of a digital tool. Low and medium severe problems do not have priority according to Nielsen and Loranger because although they are bothersome, they are not likely to directly influence the usage of a tool. This makes it uninteresting to tackle them from a cost-benefit perspective. Serious and critical severe problems on the contrary can be so disrupting that they can make users stop using a tool or prevent them from even starting to use it at all. Therefore, they should not be ignored during the development process of a digital tool [<xref ref-type="bibr" rid="ref41">41</xref>].</p>
        <p>As a next step, researcher MW started open coding of all fragments in the transcripts of the semistructured interviews that were related to (dis)satisfaction about the ease of use of the questionnaire using MAXQDA 10 (VERBI Software). After she finished open coding, she organized and structured the codes until a coding scheme emerged on the basis of which the part of the research question that was related to satisfaction of the participants could be answered sufficiently.</p>
        <p>As a last step, researcher MW ordered the analyzed data into 3 groups: data of participants who had (1) no, (2) little, and (3) average/high experience in using mobile technology. This was done to see whether or not data differed within and between these groups.</p>
        <p>During the whole course of the study procedures, coding, analysis steps, and interpretation decisions were discussed with researchers HW, MJW, and WD.</p>
      </sec>
      <sec>
        <title>Ethics</title>
        <p>No external funding was received by the Utrecht University of Applied Sciences to conduct this study. This study was submitted to the medical ethics committee of the Academic Medical Centre of Amsterdam which declared that it does not fall under the scope of the <italic>Medical Research Involving Human Subjects Act</italic>. The study was conducted according to the principles of the Declaration of Helsinki. All participants provided written informed consent. The participants’ names used in this article are all fictitious to protect their privacy.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Effectiveness</title>
        <p>Overall, 9 out of the 24 participants in this study did not complete the DTTSQ fully (see <xref ref-type="table" rid="table3">Table 3</xref>). Michelle (56 years), Bill (72 years), and Helga (54 years), who were all inexperienced in using mobile technology stopped completing the questionnaire by using the stop button. Inexperienced Ida (66 years), Ilene (79 years), Dora (77 years), and Mia (73 years) and little-experienced Peter (18 years) and Rose (60 years) went through the whole questionnaire but unintentionally left one or more parts open.</p>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Experience with mobile technology and completion of the Dutch Talking Touch Screen Questionnaire.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="605"/>
            <col width="219"/>
            <col width="176"/>
            <thead>
              <tr valign="top">
                <td>(Sub)Population</td>
                <td>Not fully completed</td>
                <td>Fully completed</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>No experience using mobile technology (n=9)</td>
                <td>7</td>
                <td>2</td>
              </tr>
              <tr valign="top">
                <td>Little experience using mobile technology (n=11)</td>
                <td>2</td>
                <td>9</td>
              </tr>
              <tr valign="top">
                <td>Average/high experience using mobile technology (n=4)</td>
                <td>—</td>
                <td>4</td>
              </tr>
              <tr valign="top">
                <td>Total population (N=24)</td>
                <td>9</td>
                <td>15</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Unanswered (Parts of) Questions</title>
        <p>Inexperienced Michelle (56 years), Ida (66 years), Ilene (79 years), Dora (77 years), and Mia (73 years) and little-experienced Peter (18 years) and Rose (60 years) failed to fully complete the DTTSQ because they failed to select answering options and/or unintentionally skipped questions by double-tapping on the next button (see problems 1-5 in <xref ref-type="table" rid="table4">Table 4</xref>). All participants, except for Michelle, additionally failed to notice that they had not effectively selected an answer because the difference between activated and nonactivated answers was not accentuated enough (see problem 6 in <xref ref-type="table" rid="table4">Table 4</xref>).</p>
      </sec>
      <sec>
        <title>Use of the Stop Button</title>
        <p>When inexperienced Michelle (56 years) noticed most of her answers were missing from the summary in question 6, she got confused. In question 6, she was asked to choose the 3 most important activities in which she was limited. The screen contained only 1 activity photo whereas, in her mind, she had selected a lot of photo’s earlier. Except for the 1 photo that she had managed to select, she had tapped on the text beneath the photos, in which case, the item was not activated (see problem 5 in <xref ref-type="table" rid="table4">Table 4</xref>). The activity on the 1 photo that she had managed to select was of no priority to her. Therefore, she decided to use the stop button and ended the questionnaire.</p>
        <p>Inexperienced Bill (72 years) had a lot of trouble operating the questionnaire. He commented on the introduction clip:</p>
        <disp-quote>
          <p>I do not think that what she is saying is difficult, but I just am not able to remember it. I have no experience with these kind of devices. So I forgot what she said right away.</p>
        </disp-quote>
        <p>Bill managed to get to question 4 by activating the help function on each screen he entered. When he touched the navigation button to see all the activity photos in question 4, the photo gallery moved in a different direction then he had presumed. This startled him somewhat and made him forget that he had to push the next button to go to the next screen (see problem 7 in <xref ref-type="table" rid="table4">Table 4</xref>). He activated the help function again, but that was of no use anymore. After trying a few buttons without succeeding to go to the next screen, he gave up and tapped on the stop button.</p>
        <p>Inexperienced Helga (54 years) operated the digital questionnaire fluently until she had to choose the 3 activities that were most important to her in question 5. She did not use the navigation function of the photo gallery and as a result she did not see all her earlier selected activities (see problem 4 in <xref ref-type="table" rid="table4">Table 4</xref>). She chose the 3 most important activities out of the 5 photos that were immediately visible. When she realized what happened, she wanted to pause for a moment to find out how she could change her answer. She interpreted the stop button as a <italic>time-out function</italic> and was a bit shocked when she found out that she had stopped the questionnaire altogether.</p>
        <p>A complete overview of frequency and severity of all problems encountered can be found in <xref ref-type="table" rid="table4">Table 4</xref>.</p>
        <table-wrap position="float" id="table4">
          <label>Table 4</label>
          <caption>
            <p>Frequency and severity of encountered problems during the completion processes of all participants.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="450"/>
            <col width="210"/>
            <col width="150"/>
            <col width="190"/>
            <thead>
              <tr valign="top">
                <td>Problem</td>
                <td>Number of participants</td>
                <td>Frequency</td>
                <td>Severity rating</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1. Accidently skipping a screen by double tapping on the next button</td>
                <td>8</td>
                <td>16</td>
                <td>Serious</td>
              </tr>
              <tr valign="top">
                <td>2. Double-tap on answering option causing activation and deactivation of the answer of choice</td>
                <td>1</td>
                <td>1</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>3. Skipping a screen by accidently touching the next button with the palm of the hand</td>
                <td>1</td>
                <td>1</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>4. Not using the navigation function of the photo gallery in question 4 causing the participant not seeing all presented response items</td>
                <td>2</td>
                <td>2</td>
                <td>Medium</td>
              </tr>
              <tr valign="top">
                <td>5. Touching the text underneath a photo in question 4 to select an activity instead of touching the photo itself causing the activity not to be selected</td>
                <td>3</td>
                <td>30</td>
                <td>Serious</td>
              </tr>
              <tr valign="top">
                <td>6. Not able to see whether or not a selected answer is activated (not accentuated enough)</td>
                <td>8</td>
                <td>8</td>
                <td>Medium</td>
              </tr>
              <tr valign="top">
                <td>7. Not knowing how to get to the next screen</td>
                <td>1</td>
                <td>1</td>
                <td>Medium</td>
              </tr>
              <tr valign="top">
                <td>8. Pushing too hard or tapping too soft on the touch screen causing the touch screen not to respond</td>
                <td>11</td>
                <td>40</td>
                <td>Serious</td>
              </tr>
              <tr valign="top">
                <td>9. Not able to correct a wrong answer</td>
                <td>8</td>
                <td>13</td>
                <td>Serious</td>
              </tr>
              <tr valign="top">
                <td>10. Not reading the text above the photos of question 5 causing the participant to keep on performing the task given with question 4</td>
                <td>4</td>
                <td>8</td>
                <td>Medium</td>
              </tr>
              <tr valign="top">
                <td>11. Not noticing that the multiple numerical rating scale-effort scores in question 8 are related to different activities, which by mistake results in identical scores for different activities</td>
                <td>1</td>
                <td>1</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>12. Scoring the body chart in question 2 mirrored</td>
                <td>2</td>
                <td>2</td>
                <td>Low</td>
              </tr>
              <tr valign="top">
                <td>13. Scoring (serial) questions that do not apply to the participants’ situation (forced by the software)</td>
                <td>1</td>
                <td>4</td>
                <td>Medium</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Number of Problems</title>
        <p>Average-/high-experienced Ellen (32 years), Sandra (39 years), and Joline (19 years) and little-experienced Jill (55 years), Lydia (56 years), and Christine (39 years) were able to complete the questionnaire without any problems. The other 18 participants were not able to operate the questionnaire fluently. In an absolute as well as relative sense, more participants with no experience in using mobile technology encountered problems during the completion of the DTTSQ than little-experienced participants did (see <xref ref-type="table" rid="table5">Table 5</xref>). Inexperienced participants encountered an average of 3 problems per person, whereas participants with little experience encountered an average of 2 problems per person. Within the subgroup of average-/high-experience participants, only 1 person encountered ` problem during completion (see <xref ref-type="table" rid="table5">Table 5</xref>). A total of 11 participants encountered problem 8, “Pushing too hard or tapping too soft on the touch screen causing the touch screen not to respond” multiple times (see <xref ref-type="table" rid="table4">Table 4</xref>). In some case, participants looked startled after problem 8 occurred. In these cases, researcher IT encouraged the participant to go on by kindly saying <italic>try again</italic>.</p>
        <table-wrap position="float" id="table5">
          <label>Table 5</label>
          <caption>
            <p>Number of participants encountering each problem per level of experience with using mobile technology (N=24).</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="90"/>
            <col width="200"/>
            <col width="240"/>
            <col width="310"/>
            <col width="160"/>
            <thead>
              <tr valign="top">
                <td>Problem</td>
                <td>No experience (n=9)</td>
                <td>Little experience (n=11)</td>
                <td>Average/high experience (n=4)</td>
                <td>Total population</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>1</td>
                <td>5</td>
                <td>3</td>
                <td>—<sup>a</sup></td>
                <td>8</td>
              </tr>
              <tr valign="top">
                <td>2</td>
                <td>1</td>
                <td>—</td>
                <td>—</td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>3</td>
                <td>—</td>
                <td>1</td>
                <td>—</td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>4</td>
                <td>1</td>
                <td>1</td>
                <td>—</td>
                <td>2</td>
              </tr>
              <tr valign="top">
                <td>5</td>
                <td>2</td>
                <td>1</td>
                <td>—</td>
                <td>3</td>
              </tr>
              <tr valign="top">
                <td>6</td>
                <td>4</td>
                <td>4</td>
                <td>—</td>
                <td>8</td>
              </tr>
              <tr valign="top">
                <td>7</td>
                <td>1</td>
                <td>—</td>
                <td>—</td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>8</td>
                <td>6</td>
                <td>5</td>
                <td>—</td>
                <td>11</td>
              </tr>
              <tr valign="top">
                <td>9</td>
                <td>3</td>
                <td>4</td>
                <td>1</td>
                <td>8</td>
              </tr>
              <tr valign="top">
                <td>10</td>
                <td>2</td>
                <td>2</td>
                <td>—</td>
                <td>4</td>
              </tr>
              <tr valign="top">
                <td>11</td>
                <td>1</td>
                <td>—</td>
                <td>—</td>
                <td>1</td>
              </tr>
              <tr valign="top">
                <td>12</td>
                <td>1</td>
                <td>1</td>
                <td>—</td>
                <td>2</td>
              </tr>
              <tr valign="top">
                <td>13</td>
                <td>—</td>
                <td>1</td>
                <td>—</td>
                <td>1</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table5fn1">
              <p><sup>a</sup>Not applicable.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Efficiency</title>
        <p>The 21 participants who got to the end of the questionnaire had an average completion time of 10 min and 25 seconds. Inexperienced participants needed more time than little-experienced participants did, who in their turn needed more time than average-/high-experienced participants did (see <xref ref-type="table" rid="table6">Table 6</xref>).</p>
        <table-wrap position="float" id="table6">
          <label>Table 6</label>
          <caption>
            <p>Completion time of all participants who did not end the questionnaire prematurely.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="450"/>
            <col width="170"/>
            <col width="180"/>
            <col width="200"/>
            <thead>
              <tr valign="top">
                <td>(Sub)Population</td>
                <td>Mean completion time (min)</td>
                <td>Median completion time (min)</td>
                <td>Range of completion times (min)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>No experience with mobile technology (n=6)</td>
                <td>11.38</td>
                <td>9.38</td>
                <td>8.2 to 22.10</td>
              </tr>
              <tr valign="top">
                <td>Little experience with mobile technology (n=11)</td>
                <td>10.41</td>
                <td>9.57</td>
                <td>6.54 to 18.10</td>
              </tr>
              <tr valign="top">
                <td>Average/high experience with mobile technology (n=4)</td>
                <td>7. 55</td>
                <td>7.42</td>
                <td>5.50 to 10.26</td>
              </tr>
              <tr valign="top">
                <td>Total population (n=21)</td>
                <td>10.25</td>
                <td>9.43</td>
                <td>5.50 to 22.10</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
      </sec>
      <sec>
        <title>Satisfaction</title>
        <p>All participants were satisfied with the ease of use of the questionnaire. The use of plain language, the way ICT was used, and the way the user interface was designed were greatly appreciated by the participants:</p>
        <disp-quote>
          <p>Everything was well described. I am not always able to understand everything, but this went well. I understood what was asked of me.</p>
          <attrib>Inexperienced Dora, 77 years</attrib>
        </disp-quote>
        <disp-quote>
          <p>I have trouble operating my mobile phone and I own a notebook but don’t you ask me how that thing works! I am capable of a lot but I am not technical in that way. [...] This was the first time for me to use a tablet computer. I only had to follow the instructions. I did not have to start it up or open something, it just started working and it shut down by itself. I thought it was easy to work with. Better than when you have to write things down.</p>
          <attrib>Little-experienced Roger, 70 years</attrib>
        </disp-quote>
        <disp-quote>
          <p>I am a very visual person. And this thing is very visual. [...] Like green is ‘no pain’ and red is ‘a lot of pain'.</p>
          <attrib>Average/high experienced Ellen, 32 years</attrib>
        </disp-quote>
        <p>All participants were satisfied with the completion time of the DTTSQ.</p>
      </sec>
      <sec>
        <title>Satisfied Despite Encountering Problems</title>
        <p>Operation problems, regardless of the amount and severity of the problems encountered by each individual participant, did not influence satisfaction about the ease of use of the questionnaire. Little-experienced Francine (61 years), for instance, was asked how she felt about the fact that the app did not always respond to her touch right away (see problem 8 in <xref ref-type="table" rid="table4">Table 4</xref>). She encountered this problem 13 times in total. She lightheartedly answered as follows:</p>
        <disp-quote>
          <p>
            <italic>Oh these are things that happen. I experience the same things with my own computer. My computer refuses to sometimes, so... I think I was just pushing too hard on the tablet sometimes, that’s all.</italic>
          </p>
        </disp-quote>
        <p>When inexperienced Bill (72 years), who used the stop button, was asked if he would have preferred a paper-based questionnaire he said the following:</p>
        <disp-quote>
          <p>
            <italic>No. It took me some time to get used to it but it is easy to use actually.</italic>
          </p>
        </disp-quote>
      </sec>
      <sec>
        <title>Expectations Exceeded</title>
        <p>A total of 9 participants explicitly stated that operating the questionnaire was easier than they had expected beforehand. When inexperienced Ida (66 years) was confronted with the questionnaire she agitatedly said the following:</p>
        <disp-quote>
          <p>
            <italic>Never in a million years I believe I can do this. That I can tell you right away.</italic>
          </p>
        </disp-quote>
        <p>Noticeably reluctant and nervous, she started to complete the questionnaire. When she finished, she seemed surprised and relieved. She smiled and said the following:</p>
        <disp-quote>
          <p>
            <italic>Okay? So this was the questionnaire? [...] Ooooh but this was doable! I thought I would have to look up things and operate it like my grandchildren do.</italic>
          </p>
        </disp-quote>
        <p>And then she started laughing out loud and cheerfully asked if anyone would like to have some coffee.</p>
        <p>Little-experienced Christine (39 years) was positively surprised too:</p>
        <disp-quote>
          <p>
            <italic>It responds really well. Normally I am not that good with screens, but this is easy. It almost feels like a game! It really responds nicely. Nothing disappears when I touch it. It reacts very calmly but at the same time it is very fast. I really like that it contains photo’s instead of drawings. It is instantly clear: these are my activities and that is what they mean by “sitting down”. You see it right away. I also like the regular summaries. It keeps you on track and enables you to check whether or not you forgot something.</italic>
          </p>
        </disp-quote>
      </sec>
      <sec>
        <title>Participants’ Recommendations for Improvement</title>
        <p>The most mentioned recommendations for improvement of the usability of the DTTSQ by participants were: shorten the length of the instructions, accentuate the activated response items, and improve the user interface of question 4 by giving participants a complete overview of activities to choose from in one screen, without having to use complicated navigation functions.</p>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>All participants within this study were very satisfied with the ease of use of the DTTSQ. Overall, 9 participants stated that the usability of the app exceeded their expectations. The participants who had no experience with using mobile technology completed the prototype of the DTTSQ less effectively and efficiently than the little- and average-/high-experienced participants did. In the group of average-/high-experienced participants, only 1 problem was encountered in total, whereas the inexperienced participants encountered an average of 3 and the little-experienced participants an average of 2 problems per person. Overall, 13 different kind of problems were encountered during this study. From a cost-benefit perspective, 4 of these problems will need to be addressed during future development of the DTTSQ because they have the potential to influence the future usage of the tool negatively [<xref ref-type="bibr" rid="ref41">41</xref>]. The 4 problems that need to be addressed are problem 1 “Accidently skipping a screen by double tapping on the next button,” problem 5 “Touching the text underneath a photo in question 4 to select an activity instead of touching the photo itself causing the activity not to be selected,” problem 8 “Pushing too hard or tapping too soft on the touch screen causing the touch screen not to respond,” and problem 9 “Not able to correct a wrong answer.” Participants also recommended to shorten the length of the instructions and improve the user interface of question 4 by giving participants a complete overview of activities to choose from in one screen, without having to use complicated navigation functions.</p>
      </sec>
      <sec>
        <title>Comparison With Previous Work</title>
        <p>In earlier studies, talking touch screens were found to be easy to use for people with different levels of education, literacy, or digital skills. These conclusions were based on study participants’ level of satisfaction with the ease of use of the tool [<xref ref-type="bibr" rid="ref44">44</xref>,<xref ref-type="bibr" rid="ref45">45</xref>] or on results on satisfaction combined with the efficiency with which the tool was completed [<xref ref-type="bibr" rid="ref46">46</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]. Effectiveness was not, or in case of Vargas et al very slightly [<xref ref-type="bibr" rid="ref45">45</xref>], tested. This is a debatable approach, because Frokjaer et al consider effectiveness, efficiency, and satisfaction as independent aspects of usability and state that it is risky to assume that there are correlations between these aspects [<xref ref-type="bibr" rid="ref34">34</xref>]. Therefore, according to Frokjaer et al, satisfaction and efficiency outcomes should always be tested in combination with outcomes of effectiveness to give a complete and realistic overview of the usability of a tool. The results of this study confirm the necessity of combining all 3 aspects of usability during usability studies. All participants in this study, including participants who were not able to fully complete the questionnaire because of problems they had with operating the app, were satisfied with the usability of the DTTSQ. Looking solely at the results on satisfaction with the ease of use (which were also found in the comparable studies [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]) one could make the assumption that the DTTSQ is, usability wise, ready to be released. Looking at the data found on efficiency within this study, one can see that more-experienced participants need less time to complete the questionnaire. This seems logical and matches the results of comparable studies [<xref ref-type="bibr" rid="ref46">46</xref>,<xref ref-type="bibr" rid="ref49">49</xref>]. In addition, the completion time was acceptable to all participants of this study. On the basis of the efficiency results solely, one could also conclude that the DTTSQ was ready to be released. Looking at the results on effectiveness and specifically at the severity rates of the problems that occurred during the response process though, the researchers of this study concluded that the usability of the DTTSQ needs to be improved to prevent problem 1, 5, 8, and 9 from occurring before it can be released.</p>
        <p>The results of this study show how difficult it is to strive for an <italic>inclusive design</italic>. A lot of effort was put into developing a tool that is easy to use for potential users at risk of exclusion from usage of mHealth tools [<xref ref-type="bibr" rid="ref31">31</xref>]. By choosing a co-design strategy, development of a user-friendly tool for people with diverse levels of education, literacy, and digital skills was taken a step further than what was done in earlier comparable projects [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]. In the other projects, users were involved in the evaluation process of the tools, but development was done by designers and health professionals. In spite of the user-centered development approach that was taken during the development process of the DTTSQ, the goal of inclusive design was not reached yet. Looking at the results of this study, the tool is ready to be released for average-/high-experienced users but not for less-experienced future users. To be able to evaluate the worth of including potential users at risk of exclusion, it would be interesting to be able to compare data on efficiency and effectiveness of talking touch screens that were developed earlier. Specifically, because the user interface and structure of the DTTSQ differs from comparable tools. For instance, the screen of the DTTSQ contains fewer buttons and operation functions, it does not have a back function, it provides summaries of given answers regularly to the respondent, and questions are not automatically read out loud. In addition, the design and format of the answering options in the earlier developed talking touch screen [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>] does not match the recommendations given by the low-literate people that helped to design the DTTSQ [<xref ref-type="bibr" rid="ref31">31</xref>]. If it would be possible to compare results on effectiveness from the tests of several different kind of talking touch screens, a lot of insight could be gained in what does and does not work in striving for an inclusive design for less-skilled users of such tools.</p>
        <p>According to Frokjaer et al, relations between the 3 aspects of usability depend in complex ways on the app domain, use context, and user’s experience [<xref ref-type="bibr" rid="ref34">34</xref>]. User’s experience may well have been of influence on the satisfaction outcomes of this study. Overall, 83% (20/24) of the total study population had no or little experience in using mobile technology (see <xref ref-type="table" rid="table1">Tables 1</xref> and <xref ref-type="table" rid="table2">2</xref>). Limited or no user experience may have caused a form of computer anxiety, resulting in low self-efficacy, which in turn led to low expectations toward the ease of use of the DTTSQ [<xref ref-type="bibr" rid="ref51">51</xref>]. A total of 9 out of the 24 participants in this study explicitly stated that operating the DTTSQ was easier than they had expected beforehand. The other participants did not explicitly state this, but their statements on the ease of use could easily be interpreted as such. No participant stated or gave the impression that the ease of use of the DTTSQ was lower than they would have expected. According to the Expectation Confirmation Theory [<xref ref-type="bibr" rid="ref52">52</xref>], actual performance exceeding the expectations of testers leads to satisfaction among these testers. The more their expectations are exceeded, the more satisfied testers will become. Owing to the limited user experience of most of the study participants, expectations toward the ease of use of the DTTSQ may have been low, which may have made it easier to exceed them. Especially considering that the DTTSQ was specifically designed to be easy to use for low-educated people who lack the necessary skills to use ICT [<xref ref-type="bibr" rid="ref31">31</xref>]. Looking at the results of studies that evaluated the satisfaction about the ease of use of earlier developed talking touch screens, a similar picture of highly satisfied study participants emerges [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]. The qualitative results in 2 of these studies also show that participants’ expectations regarding the ease of use of the tested tool were exceeded [<xref ref-type="bibr" rid="ref44">44</xref>,<xref ref-type="bibr" rid="ref47">47</xref>] and 2 other authors report that satisfaction among the study participants was <italic>extremely</italic> and <italic>overwhelmingly</italic> high [<xref ref-type="bibr" rid="ref45">45</xref>,<xref ref-type="bibr" rid="ref48">48</xref>]. In all of the comparable studies, a large proportion of the study participants had no or limited computer experience [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]. It is reasonable to assume that limited computer experience may have led to low expectations regarding the ease of use of the talking touch screens and, therefore, played a role in the high satisfaction outcomes.</p>
      </sec>
      <sec>
        <title>Strengths and Limitations</title>
        <p>It is a strength of this study that all 3 aspects of usability, instead of just satisfaction and efficiency, were thoroughly tested and that all of the results of the tests were differentiated for inexperienced and little- and average-/high-experienced users (which was not the case in the reports of the comparable studies [<xref ref-type="bibr" rid="ref44">44</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]). To this date, this is the first study on usability of talking touch screens that has taken this approach. As a result, an insight was gained into what kind and amount of usability problems are encountered by the most vulnerable group of potential users.</p>
        <p>It is a strength in itself that inexperienced as well as little and average-/high-experienced users of mobile technology were included in this study. Although recommended in the literature [<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref53">53</xref>], to this date, there has been an insufficient number of empirical studies to prove the worth of involving future users at risk of exclusion in the development process of eHealth tools [<xref ref-type="bibr" rid="ref54">54</xref>]. In a recent review, Latulippe et al found only 3 studies that involved future users at risk of exclusion in their design and evaluation processes [<xref ref-type="bibr" rid="ref8">8</xref>]. This study contributes to the body of knowledge of inclusive mHealth design which involves active participation of vulnerable potential users in usability evaluation.</p>
        <p>The qualitative TSTI method [<xref ref-type="bibr" rid="ref32">32</xref>] was chosen for data collection in this study. This method was never used in a usability study before. The results of this study show that the TSTI method is suitable to gain insight into the usability of mHealth tools. It helped the researchers to understand not only what kind of usability problems occurred but also what caused these problems to occur and what effect encountering the problems had on participants. In addition, this method suited the needs of low-educated and low-literate participants by not demanding any reading or writing skills from them. A downside of the chosen method is the lack of generalizability of the data.</p>
        <p>A limitation of this study was that participants were encouraged by the interviewer to try touching the screen again when they looked startled because it did not react to their initial touch. This may have influenced the results on effectiveness because it is unknown what would have happened if the interviewer would not have interfered. This may vary from no effect because the participant would have tried it again anyway, to a higher frequency of occurrence of problem 8, to more participants prematurely stopping to complete the DTTSQ because of being under the impression that the app had stopped working. Any kind of interference in the process of usability testing has a direct influence on the effectiveness results and possibly also on the efficiency and satisfaction results and should therefore be avoided.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>The usability of the DTTSQ needs to be improved before it can be released. No problems were found with satisfaction or efficiency during the usability test. Effectiveness needs to be enhanced by (1) making it easier to navigate through screens without the possibility of accidently skipping one, (2) enabling the possibility to insert an answer by tapping on the text underneath a photograph instead of just touching the photograph itself, and (3) making it easier to correct wrong answers. Participants additionally recommended to minimize the length of the instructions and present all the answering options of question 4 in one screen.</p>
      </sec>
      <sec>
        <title>Directions for Future Research</title>
        <p>During further development of the DTTSQ, both the results of this study and the study on response process of the DTTSQ [<xref ref-type="bibr" rid="ref36">36</xref>] should be taken into account simultaneously. The usability and the response processes will have to be retested in exactly the same manner after adjustments in the DTTSQ have been made. This process will have to be repeated until an acceptable level of usability and face validity of the DTTSQ are reached. The next step in research should be quantitative usability, validity, and reliability testing producing generalizable data.</p>
        <p>Considering the difference in the number of problems encountered by inexperienced and little-experienced participants versus average-/high-experienced participants within this study, it can be concluded that in striving for an inclusive design, it is vital to involve potential users at risk of exclusion during further development and testing of the DTTSQ. Selecting quantitative methods for this purpose may be quite challenging because the researchers will have to develop a quantitative study design that will enable people with low literacy skills and low educational levels to participate. Research designs that include reading and writing tasks for participants are ineligible because these tasks may lead to exclusion of these vulnerable and hard-to-reach populations [<xref ref-type="bibr" rid="ref55">55</xref>].</p>
        <p>Researchers who want to investigate the usability of mHealth tools in populations that include little-experienced or inexperienced participants should take into account that the expectations of these participants may easily be exceeded resulting in high participant satisfaction outcomes regardless of the effectiveness and efficiency with which the tool is used. Satisfaction outcomes are influenced by the expectations that participants have before the test. It could be interesting to measure and further investigate computer anxiety and self-efficacy toward the use of the tested tool before and after usability testing to be able to put satisfaction outcomes into perspective.</p>
        <p>Further research is necessary to gain more insight into the needs, preferences, capacities, values, and goals in relation to mHealth technology of people with low literacy skills, low educational levels, and no or little experience with using mobile technology. Insight is also needed into what effects meeting these user requirements will have on actual future use of these tools by these specific populations.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>An overview of the screenshots of the Dutch Talking Touch Screen Questionnaire.</p>
        <media xlink:href="formative_v3i3e11617_app1.pdf" xlink:title="PDF File  (Adobe PDF File)1288 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">DTTSQ</term>
          <def>
            <p>Dutch Talking Touch Screen Questionnaire</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">eHealth</term>
          <def>
            <p>electronic health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">ICT</term>
          <def>
            <p>information and communication technology</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">mHealth</term>
          <def>
            <p>mobile health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">TSTI</term>
          <def>
            <p>Three-Step Test-Interview</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The authors would like to thank Anita Cremers, PhD, for her contribution to reflecting on the data from a designer perspective and Kees van der Veer, PhD, and Jan Pool, PhD, for helping them to think about the application of the TSTI method in low-educated populations.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>McGregor</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Brunner</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Keep</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Janssen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Barnet</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>What is ehealth (6)? Development of a conceptual model for ehealth: qualitative study with key informants</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <month>10</month>
          <day>24</day>
          <volume>19</volume>
          <issue>10</issue>
          <fpage>e324</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2017/10/e324/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.8106</pub-id>
          <pub-id pub-id-type="medline">29066429</pub-id>
          <pub-id pub-id-type="pii">v19i10e324</pub-id>
          <pub-id pub-id-type="pmcid">PMC5676031</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Granja</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Janssen</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Johansen</surname>
              <given-names>MA</given-names>
            </name>
          </person-group>
          <article-title>Factors determining the success and failure of ehealth interventions: systematic review of the literature</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>05</month>
          <day>1</day>
          <volume>20</volume>
          <issue>5</issue>
          <fpage>e10235</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/5/e10235/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/10235</pub-id>
          <pub-id pub-id-type="medline">29716883</pub-id>
          <pub-id pub-id-type="pii">v20i5e10235</pub-id>
          <pub-id pub-id-type="pmcid">PMC5954232</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Flodgren</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Rachas</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Farmer</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Inzitari</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Shepperd</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Interactive telemedicine: effects on professional practice and health care outcomes</article-title>
          <source>Cochrane Database Syst Rev</source>
          <year>2015</year>
          <month>09</month>
          <day>7</day>
          <issue>9</issue>
          <fpage>CD002098</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26343551"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/14651858.CD002098.pub2</pub-id>
          <pub-id pub-id-type="medline">26343551</pub-id>
          <pub-id pub-id-type="pmcid">PMC6473731</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Henderson</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Knapp</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Fernández</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Beecham</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Hirani</surname>
              <given-names>SP</given-names>
            </name>
            <name name-style="western">
              <surname>Cartwright</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rixon</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Beynon</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rogers</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bower</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Doll</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Fitzpatrick</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Steventon</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Bardsley</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hendy</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Newman</surname>
              <given-names>SP</given-names>
            </name>
            <collab>Whole System Demonstrator Evaluation Team</collab>
          </person-group>
          <article-title>Cost effectiveness of telehealth for patients with long term conditions (whole systems demonstrator telehealth questionnaire study): nested economic evaluation in a pragmatic, cluster randomised controlled trial</article-title>
          <source>Br Med J</source>
          <year>2013</year>
          <month>03</month>
          <day>20</day>
          <volume>346</volume>
          <fpage>f1035</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.bmj.com/cgi/pmidlookup?view=long&#38;pmid=23520339"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmj.f1035</pub-id>
          <pub-id pub-id-type="medline">23520339</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ramirez</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Johnson</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Gonzalez</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ramirez</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Rubino</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Rossetti</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Assessing the use of mobile health technology by patients: an observational study in primary care clinics</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2016</year>
          <month>04</month>
          <day>19</day>
          <volume>4</volume>
          <issue>2</issue>
          <fpage>e41</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mhealth.jmir.org/2016/2/e41/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/mhealth.4928</pub-id>
          <pub-id pub-id-type="medline">27095507</pub-id>
          <pub-id pub-id-type="pii">v4i2e41</pub-id>
          <pub-id pub-id-type="pmcid">PMC4858592</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kontos</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Blake</surname>
              <given-names>KD</given-names>
            </name>
            <name name-style="western">
              <surname>Chou</surname>
              <given-names>WS</given-names>
            </name>
            <name name-style="western">
              <surname>Prestin</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Predictors of ehealth usage: insights on the digital divide from the health information national trends survey 2012</article-title>
          <source>J Med Internet Res</source>
          <year>2014</year>
          <month>07</month>
          <day>16</day>
          <volume>16</volume>
          <issue>7</issue>
          <fpage>e172</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2014/7/e172/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.3117</pub-id>
          <pub-id pub-id-type="medline">25048379</pub-id>
          <pub-id pub-id-type="pii">v16i7e172</pub-id>
          <pub-id pub-id-type="pmcid">PMC4129114</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jensen</surname>
              <given-names>JD</given-names>
            </name>
            <name name-style="western">
              <surname>King</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Guntzviller</surname>
              <given-names>LM</given-names>
            </name>
          </person-group>
          <article-title>Utilization of internet technology by low-income adults: the role of health literacy, health numeracy, and computer assistance</article-title>
          <source>J Aging Health</source>
          <year>2010</year>
          <month>09</month>
          <volume>22</volume>
          <issue>6</issue>
          <fpage>804</fpage>
          <lpage>26</lpage>
          <pub-id pub-id-type="doi">10.1177/0898264310366161</pub-id>
          <pub-id pub-id-type="medline">20495159</pub-id>
          <pub-id pub-id-type="pii">0898264310366161</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Latulippe</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Hamel</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Giroux</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Social health inequalities and ehealth: a literature review with qualitative synthesis of theoretical and empirical studies</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <month>04</month>
          <day>27</day>
          <volume>19</volume>
          <issue>4</issue>
          <fpage>e136</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2017/4/e136/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.6731</pub-id>
          <pub-id pub-id-type="medline">28450271</pub-id>
          <pub-id pub-id-type="pii">v19i4e136</pub-id>
          <pub-id pub-id-type="pmcid">PMC5427250</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mackert</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Mabry-Flynn</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Champlin</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Donovan</surname>
              <given-names>EE</given-names>
            </name>
            <name name-style="western">
              <surname>Pounders</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Health literacy and health information technology adoption: the potential for a new digital divide</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>10</month>
          <day>4</day>
          <volume>18</volume>
          <issue>10</issue>
          <fpage>e264</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/10/e264/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.6349</pub-id>
          <pub-id pub-id-type="medline">27702738</pub-id>
          <pub-id pub-id-type="pii">v18i10e264</pub-id>
          <pub-id pub-id-type="pmcid">PMC5069402</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>van Dijk</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>Digital divide research, achievements and shortcomings</article-title>
          <source>Poetics</source>
          <year>2006</year>
          <month>08</month>
          <volume>34</volume>
          <issue>4-5</issue>
          <fpage>221</fpage>
          <lpage>35</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1016/j.poetic.2006.05.004"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.poetic.2006.05.004</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>van Deursen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>van Dijk</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Internet skills and the digital divide</article-title>
          <source>New Media Soc</source>
          <year>2010</year>
          <month>12</month>
          <day>6</day>
          <volume>13</volume>
          <issue>6</issue>
          <fpage>893</fpage>
          <lpage>911</lpage>
          <pub-id pub-id-type="doi">10.1177/1461444810386774</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bacigalupe</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Askari</surname>
              <given-names>SF</given-names>
            </name>
          </person-group>
          <article-title>E-health innovations, collaboration, and healthcare disparities: developing criteria for culturally competent evaluation</article-title>
          <source>Fam Syst Health</source>
          <year>2013</year>
          <month>09</month>
          <volume>31</volume>
          <issue>3</issue>
          <fpage>248</fpage>
          <lpage>63</lpage>
          <pub-id pub-id-type="doi">10.1037/a0033386</pub-id>
          <pub-id pub-id-type="medline">24059273</pub-id>
          <pub-id pub-id-type="pii">2013-32902-002</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Morey</surname>
              <given-names>OT</given-names>
            </name>
          </person-group>
          <article-title>Digital disparities</article-title>
          <source>J Consum Health Internet</source>
          <year>2007</year>
          <month>12</month>
          <volume>11</volume>
          <issue>4</issue>
          <fpage>23</fpage>
          <lpage>41</lpage>
          <pub-id pub-id-type="doi">10.1300/J381v11n04_03</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wildenbos</surname>
              <given-names>GA</given-names>
            </name>
            <name name-style="western">
              <surname>Peute</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Jaspers</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Aging barriers influencing mobile health usability for older adults: a literature based framework (MOLD-US)</article-title>
          <source>Int J Med Inform</source>
          <year>2018</year>
          <month>06</month>
          <volume>114</volume>
          <fpage>66</fpage>
          <lpage>75</lpage>
          <pub-id pub-id-type="doi">10.1016/j.ijmedinf.2018.03.012</pub-id>
          <pub-id pub-id-type="medline">29673606</pub-id>
          <pub-id pub-id-type="pii">S1386-5056(18)30245-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Victorson</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Banas</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Smith</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Languido</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Gutierrez</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Cordero</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Flores</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>eSalud: designing and implementing culturally competent ehealth research with latino patient populations</article-title>
          <source>Am J Public Health</source>
          <year>2014</year>
          <month>12</month>
          <volume>104</volume>
          <issue>12</issue>
          <fpage>2259</fpage>
          <lpage>65</lpage>
          <pub-id pub-id-type="doi">10.2105/AJPH.2014.302187</pub-id>
          <pub-id pub-id-type="medline">25320901</pub-id>
          <pub-id pub-id-type="pmcid">PMC4232154</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>López</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <source>Pew Research Center</source>
          <year>2016</year>
          <access-date>2018-07-18</access-date>
          <comment>Digital Divide Narrows for Latinos as More Spanish Speakers and Immigrants Go Online<ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.pewhispanic.org/2016/07/20/digital-divide-narrows-for-latinos-as-more-spanish-speakers-and-immigrants-go-online/">http://www.pewhispanic.org/2016/07/20/digital-divide-narrows-for-latinos-as-more-spanish-speakers-and-immigrants-go-online/</ext-link>
                                                </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>López</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Grant</surname>
              <given-names>RW</given-names>
            </name>
          </person-group>
          <article-title>Closing the gap: eliminating health care disparities among Latinos with diabetes using health information technology tools and patient navigators</article-title>
          <source>J Diabetes Sci Technol</source>
          <year>2012</year>
          <month>01</month>
          <day>1</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>169</fpage>
          <lpage>76</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22401336"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/193229681200600121</pub-id>
          <pub-id pub-id-type="medline">22401336</pub-id>
          <pub-id pub-id-type="pmcid">PMC3320835</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <collab>World Health Organization</collab>
          </person-group>
          <source>MHealth: New Horizons for Health Through Mobile Technologies</source>
          <year>2011</year>
          <publisher-loc>Geneva, Switzerland</publisher-loc>
          <publisher-name>World Health Organization</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Arora</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Ford</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Terp</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Abramson</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Ruiz</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Camilon</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Coyne</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lam</surname>
              <given-names>CN</given-names>
            </name>
            <name name-style="western">
              <surname>Menchine</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Burner</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Describing the evolution of mobile technology usage for Latino patients and comparing findings to national mhealth estimates</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2016</year>
          <month>09</month>
          <volume>23</volume>
          <issue>5</issue>
          <fpage>979</fpage>
          <lpage>83</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26995564"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/jamia/ocv203</pub-id>
          <pub-id pub-id-type="medline">26995564</pub-id>
          <pub-id pub-id-type="pii">ocv203</pub-id>
          <pub-id pub-id-type="pmcid">PMC4997038</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hamilton</surname>
              <given-names>EC</given-names>
            </name>
            <name name-style="western">
              <surname>Saiyed</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Miller 3rd</surname>
              <given-names>CC</given-names>
            </name>
            <name name-style="western">
              <surname>Eguia</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Fonseca</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Baum</surname>
              <given-names>GP</given-names>
            </name>
            <name name-style="western">
              <surname>Tsao</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Austin</surname>
              <given-names>MT</given-names>
            </name>
          </person-group>
          <article-title>The digital divide in adoption and use of mobile health technology among caregivers of pediatric surgery patients</article-title>
          <source>J Pediatr Surg</source>
          <year>2018</year>
          <month>08</month>
          <volume>53</volume>
          <issue>8</issue>
          <fpage>1478</fpage>
          <lpage>93</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jpedsurg.2017.08.023</pub-id>
          <pub-id pub-id-type="medline">28927983</pub-id>
          <pub-id pub-id-type="pii">S0022-3468(17)30519-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Richtering</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Hyun</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Neubeck</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Coorey</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Chalmers</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Usherwood</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Peiris</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Chow</surname>
              <given-names>CK</given-names>
            </name>
            <name name-style="western">
              <surname>Redfern</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>eHealth literacy: predictors in a population with moderate-to-high cardiovascular risk</article-title>
          <source>JMIR Hum Factors</source>
          <year>2017</year>
          <month>01</month>
          <day>27</day>
          <volume>4</volume>
          <issue>1</issue>
          <fpage>e4</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://humanfactors.jmir.org/2017/1/e4/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/humanfactors.6217</pub-id>
          <pub-id pub-id-type="medline">28130203</pub-id>
          <pub-id pub-id-type="pii">v4i1e4</pub-id>
          <pub-id pub-id-type="pmcid">PMC5303199</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Amo</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Education-based gaps in ehealth: a weighted logistic regression approach</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>10</month>
          <day>12</day>
          <volume>18</volume>
          <issue>10</issue>
          <fpage>e267</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/10/e267/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.5188</pub-id>
          <pub-id pub-id-type="medline">27733329</pub-id>
          <pub-id pub-id-type="pii">v18i10e267</pub-id>
          <pub-id pub-id-type="pmcid">PMC5081480</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Peeters</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Krijgsman</surname>
              <given-names>JW</given-names>
            </name>
            <name name-style="western">
              <surname>Brabers</surname>
              <given-names>AE</given-names>
            </name>
            <name name-style="western">
              <surname>Jong</surname>
              <given-names>JD</given-names>
            </name>
            <name name-style="western">
              <surname>Friele</surname>
              <given-names>RD</given-names>
            </name>
          </person-group>
          <article-title>Use and uptake of ehealth in general practice: a cross-sectional survey and focus group study among health care users and general practitioners</article-title>
          <source>JMIR Med Inform</source>
          <year>2016</year>
          <month>04</month>
          <day>6</day>
          <volume>4</volume>
          <issue>2</issue>
          <fpage>e11</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2016/2/e11/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/medinform.4515</pub-id>
          <pub-id pub-id-type="medline">27052805</pub-id>
          <pub-id pub-id-type="pii">v4i2e11</pub-id>
          <pub-id pub-id-type="pmcid">PMC4838754</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Liang</surname>
              <given-names>TH</given-names>
            </name>
          </person-group>
          <article-title>Matrix analysis of the digital divide in eHealth services using awareness, want, and adoption gap</article-title>
          <source>J Med Internet Res</source>
          <year>2012</year>
          <month>02</month>
          <day>13</day>
          <volume>14</volume>
          <issue>1</issue>
          <fpage>e11</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2012/1/e11/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.1670</pub-id>
          <pub-id pub-id-type="medline">22329958</pub-id>
          <pub-id pub-id-type="pii">v14i1e11</pub-id>
          <pub-id pub-id-type="pmcid">PMC3374548</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zach</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Dalrymple</surname>
              <given-names>PW</given-names>
            </name>
            <name name-style="western">
              <surname>Rogers</surname>
              <given-names>ML</given-names>
            </name>
            <name name-style="western">
              <surname>Williver-Farr</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Assessing internet access and use in a medically underserved population: implications for providing enhanced health information services</article-title>
          <source>Health Info Libr J</source>
          <year>2012</year>
          <month>03</month>
          <volume>29</volume>
          <issue>1</issue>
          <fpage>61</fpage>
          <lpage>71</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1111/j.1471-1842.2011.00971.x"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/j.1471-1842.2011.00971.x</pub-id>
          <pub-id pub-id-type="medline">22335290</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="web">
          <source>StatLine - CBS</source>
          <year>2018</year>
          <access-date>2018-07-18</access-date>
          <comment>Internet; Toegang, Gebruik en Faciliteiten<ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://opendata.cbs.nl/statline/#/CBS/nl/dataset/83429NED/table?ts=1522570289276">https://opendata.cbs.nl/statline/#/CBS/nl/dataset/83429NED/table?ts=1522570289276</ext-link>
                                                </comment>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mackenbach</surname>
              <given-names>JP</given-names>
            </name>
          </person-group>
          <article-title>Persistence of social inequalities in modern welfare states: explanation of a paradox</article-title>
          <source>Scand J Public Health</source>
          <year>2017</year>
          <month>03</month>
          <volume>45</volume>
          <issue>2</issue>
          <fpage>113</fpage>
          <lpage>20</lpage>
          <pub-id pub-id-type="doi">10.1177/1403494816683878</pub-id>
          <pub-id pub-id-type="medline">28212600</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Marmot</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Friel</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bell</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Houweling</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>S</given-names>
            </name>
            <collab>Commission on Social Determinants of Health</collab>
          </person-group>
          <article-title>Closing the gap in a generation: health equity through action on the social determinants of health</article-title>
          <source>Lancet</source>
          <year>2008</year>
          <month>11</month>
          <day>8</day>
          <volume>372</volume>
          <issue>9650</issue>
          <fpage>1661</fpage>
          <lpage>9</lpage>
          <pub-id pub-id-type="doi">10.1016/S0140-6736(08)61690-6</pub-id>
          <pub-id pub-id-type="medline">18994664</pub-id>
          <pub-id pub-id-type="pii">S0140-6736(08)61690-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="web">
          <source>World Health Organization</source>
          <year>2015</year>
          <access-date>2018-07-18</access-date>
          <comment>World Report On Ageing And Health<ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://apps.who.int/iris/bitstream/handle/10665/186463/9789240694811_eng.pdf;jsessionid=D8D38853326AC0A27749998FDC88F9A8?sequence=1">http://apps.who.int/iris/bitstream/handle/10665/186463/9789240694811_eng.pdf;jsessionid=D8D38853326AC0A27749998FDC88F9A8?sequence=1</ext-link>
                         <ext-link ext-link-type="webcite" xlink:href="7107POCPi"/>                         </comment>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hahn</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Cella</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Health outcomes assessment in vulnerable populations: measurement challenges and recommendations</article-title>
          <source>Arch Phys Med Rehabil</source>
          <year>2003</year>
          <month>04</month>
          <volume>84</volume>
          <issue>4 Suppl 2</issue>
          <fpage>S35</fpage>
          <lpage>42</lpage>
          <pub-id pub-id-type="doi">10.1053/apmr.2003.50245</pub-id>
          <pub-id pub-id-type="medline">12692770</pub-id>
          <pub-id pub-id-type="pii">S0003999303001941</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cremers</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Welbie</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kranenborg</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Wittink</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Deriving guidelines for designing interactive questionnaires for low-literate persons: development of a health assessment questionnaire</article-title>
          <source>Univ Access Inf Soc</source>
          <year>2015</year>
          <month>09</month>
          <day>30</day>
          <volume>16</volume>
          <issue>1</issue>
          <fpage>161</fpage>
          <lpage>72</lpage>
          <pub-id pub-id-type="doi">10.1007/s10209-015-0431-2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hak</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>van der Veer</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Jansen</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>The three-step test-interview (TSTI): an observation-based method for pretesting self-completion questionnaires</article-title>
          <source>Surv Res Methods</source>
          <year>2008</year>
          <volume>2</volume>
          <issue>3</issue>
          <fpage>50</fpage>
          <pub-id pub-id-type="doi">10.18148/srm/2008.v2i3.1669</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="web">
          <source>World Wide Web Consortium (W3C)</source>
          <access-date>2018-07-18</access-date>
          <comment>Usability - ISO 9241 Definition<ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.w3.org/2002/Talks/0104-usabilityprocess/slide3-0.html">http://www.w3.org/2002/Talks/0104-usabilityprocess/slide3-0.html</ext-link>
                                                </comment>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Frokjaer</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hertzum</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hornbaek</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Measuring Usability: Are Effectiveness, Efficiency, and Satisfaction Really Correlated?</article-title>
          <source>Proceedings of the SIGCHI Conference on Human Factors in Computing Systems</source>
          <year>2000</year>
          <conf-name>CHI'00</conf-name>
          <conf-date>April 1-6, 2000</conf-date>
          <conf-loc>The Hague, The Netherlands</conf-loc>
          <fpage>345</fpage>
          <lpage>52</lpage>
          <pub-id pub-id-type="doi">10.1145/332040.332455</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>FD</given-names>
            </name>
            <name name-style="western">
              <surname>Bagozzi</surname>
              <given-names>RP</given-names>
            </name>
            <name name-style="western">
              <surname>Warshaw</surname>
              <given-names>PR</given-names>
            </name>
          </person-group>
          <article-title>User acceptance of computer technology: a comparison of two theoretical models</article-title>
          <source>Manag Sci</source>
          <year>1989</year>
          <month>08</month>
          <volume>35</volume>
          <issue>8</issue>
          <fpage>982</fpage>
          <lpage>1003</lpage>
          <pub-id pub-id-type="doi">10.1287/mnsc.35.8.982</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Welbie</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Wittink</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Westerman</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Topper</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Snoei</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Devillé</surname>
              <given-names>WL</given-names>
            </name>
          </person-group>
          <article-title>Using plain language and adding communication technology to an existing health-related questionnaire to help generate accurate information: qualitative study</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>04</month>
          <day>23</day>
          <volume>20</volume>
          <issue>4</issue>
          <fpage>e140</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/4/e140/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.7940</pub-id>
          <pub-id pub-id-type="medline">29685873</pub-id>
          <pub-id pub-id-type="pii">v20i4e140</pub-id>
          <pub-id pub-id-type="pmcid">PMC5938598</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sanders</surname>
              <given-names>EB</given-names>
            </name>
            <name name-style="western">
              <surname>Stappers</surname>
              <given-names>PJ</given-names>
            </name>
          </person-group>
          <article-title>Co-creation and the new landscapes of design</article-title>
          <source>CoDesign</source>
          <year>2008</year>
          <month>03</month>
          <volume>4</volume>
          <issue>1</issue>
          <fpage>5</fpage>
          <lpage>18</lpage>
          <pub-id pub-id-type="doi">10.1080/15710880701875068</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Beurskens</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>de Vet</surname>
              <given-names>HC</given-names>
            </name>
            <name name-style="western">
              <surname>Köke</surname>
              <given-names>AJ</given-names>
            </name>
          </person-group>
          <article-title>Responsiveness of functional status in low back pain: a comparison of different instruments</article-title>
          <source>Pain</source>
          <year>1996</year>
          <month>04</month>
          <volume>65</volume>
          <issue>1</issue>
          <fpage>71</fpage>
          <lpage>6</lpage>
          <pub-id pub-id-type="doi">10.1016/0304-3959(95)00149-2</pub-id>
          <pub-id pub-id-type="medline">8826492</pub-id>
          <pub-id pub-id-type="pii">0304-3959(95)00149-2</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Beurskens</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>de Vet</surname>
              <given-names>HC</given-names>
            </name>
            <name name-style="western">
              <surname>Köke</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lindeman</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>van der Heijden</surname>
              <given-names>GJ</given-names>
            </name>
            <name name-style="western">
              <surname>Regtop</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Knipschild</surname>
              <given-names>PG</given-names>
            </name>
          </person-group>
          <article-title>A patient-specific approach for measuring functional status in low back pain</article-title>
          <source>J Manipulative Physiol Ther</source>
          <year>1999</year>
          <volume>22</volume>
          <issue>3</issue>
          <fpage>144</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1016/S0031-9406(05)66397-3</pub-id>
          <pub-id pub-id-type="medline">10220712</pub-id>
          <pub-id pub-id-type="pii">S0161475499000858</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Braun</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Clarke</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Using thematic analysis in psychology</article-title>
          <source>Qual Res Psychol</source>
          <year>2006</year>
          <month>01</month>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>77</fpage>
          <lpage>101</lpage>
          <pub-id pub-id-type="doi">10.1191/1478088706qp063oa</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nielsen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Loranger</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <source>Prioritizing Web Usability</source>
          <year>2006</year>
          <publisher-loc>Berkeley, California USA</publisher-loc>
          <publisher-name>New Riders</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hattink</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Droes</surname>
              <given-names>RM</given-names>
            </name>
            <name name-style="western">
              <surname>Sikkes</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Oostra</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Lemstra</surname>
              <given-names>AW</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of the digital Alzheimer center: testing usability and usefulness of an online portal for patients with dementia and their carers</article-title>
          <source>JMIR Res Protoc</source>
          <year>2016</year>
          <month>07</month>
          <day>21</day>
          <volume>5</volume>
          <issue>3</issue>
          <fpage>e144</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchprotocols.org/2016/3/e144/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/resprot.5040</pub-id>
          <pub-id pub-id-type="medline">27444209</pub-id>
          <pub-id pub-id-type="pii">v5i3e144</pub-id>
          <pub-id pub-id-type="pmcid">PMC4974452</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nielsen</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <source>Nielsen Norman Group</source>
          <year>1994</year>
          <access-date>2018-07-18</access-date>
          <comment>Severity Ratings for Usability Problems<ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.nngroup.com/articles/how-to-rate-the-severity-of-usability-problems/">https://www.nngroup.com/articles/how-to-rate-the-severity-of-usability-problems/</ext-link>
                                                </comment>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hahn</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Du</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Garcia</surname>
              <given-names>SF</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>SW</given-names>
            </name>
            <name name-style="western">
              <surname>Lai</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Victorson</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Cella</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Literacy-fair measurement of health-related quality of life will facilitate comparative effectiveness research in Spanish-speaking cancer outpatients</article-title>
          <source>Med Care</source>
          <year>2010</year>
          <month>06</month>
          <volume>48</volume>
          <issue>6 Suppl</issue>
          <fpage>S75</fpage>
          <lpage>82</lpage>
          <pub-id pub-id-type="doi">10.1097/MLR.0b013e3181d6f81b</pub-id>
          <pub-id pub-id-type="medline">20473208</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vargas</surname>
              <given-names>PA</given-names>
            </name>
            <name name-style="western">
              <surname>Robles</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Harris</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Radford</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Using information technology to reduce asthma disparities in underserved populations: a pilot study</article-title>
          <source>J Asthma</source>
          <year>2010</year>
          <month>10</month>
          <volume>47</volume>
          <issue>8</issue>
          <fpage>889</fpage>
          <lpage>94</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/20846082"/>
          </comment>
          <pub-id pub-id-type="doi">10.3109/02770903.2010.497887</pub-id>
          <pub-id pub-id-type="medline">20846082</pub-id>
          <pub-id pub-id-type="pmcid">PMC3007599</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hahn</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Cella</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Dobrez</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Shiomoto</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Marcus</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>SG</given-names>
            </name>
            <name name-style="western">
              <surname>Vohra</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Chang</surname>
              <given-names>CH</given-names>
            </name>
            <name name-style="western">
              <surname>Wright</surname>
              <given-names>BD</given-names>
            </name>
            <name name-style="western">
              <surname>Linacre</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Weiss</surname>
              <given-names>BD</given-names>
            </name>
            <name name-style="western">
              <surname>Valenzuela</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Chiang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Webster</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>The talking touchscreen: a new approach to outcomes assessment in low literacy</article-title>
          <source>Psychooncology</source>
          <year>2004</year>
          <month>02</month>
          <volume>13</volume>
          <issue>2</issue>
          <fpage>86</fpage>
          <lpage>95</lpage>
          <pub-id pub-id-type="doi">10.1002/pon.719</pub-id>
          <pub-id pub-id-type="medline">14872527</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hahn</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Cellal</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Dobrez</surname>
              <given-names>DG</given-names>
            </name>
            <name name-style="western">
              <surname>Shiomoto</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>SG</given-names>
            </name>
            <name name-style="western">
              <surname>Galvez</surname>
              <given-names>AG</given-names>
            </name>
            <name name-style="western">
              <surname>Diaz</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Valenzuela</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Chiang</surname>
              <given-names>HL</given-names>
            </name>
            <name name-style="western">
              <surname>Khan</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hudgens</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Du</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Quality of life assessment for low literacy Latinos: a new multimedia program for self-administration</article-title>
          <source>J Oncol Manag</source>
          <year>2003</year>
          <volume>12</volume>
          <issue>5</issue>
          <fpage>9</fpage>
          <lpage>12</lpage>
          <pub-id pub-id-type="medline">14619989</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Yost</surname>
              <given-names>KJ</given-names>
            </name>
            <name name-style="western">
              <surname>Webster</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Baker</surname>
              <given-names>DW</given-names>
            </name>
            <name name-style="western">
              <surname>Jacobs</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Anderson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hahn</surname>
              <given-names>EA</given-names>
            </name>
          </person-group>
          <article-title>Acceptability of the talking touchscreen for health literacy assessment</article-title>
          <source>J Health Commun</source>
          <year>2010</year>
          <volume>15</volume>
          <issue>Suppl 2</issue>
          <fpage>80</fpage>
          <lpage>92</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/20845195"/>
          </comment>
          <pub-id pub-id-type="doi">10.1080/10810730.2010.500713</pub-id>
          <pub-id pub-id-type="medline">20845195</pub-id>
          <pub-id pub-id-type="pii">926945314</pub-id>
          <pub-id pub-id-type="pmcid">PMC3269098</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Thumboo</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Wee</surname>
              <given-names>HL</given-names>
            </name>
            <name name-style="western">
              <surname>Cheung</surname>
              <given-names>YB</given-names>
            </name>
            <name name-style="western">
              <surname>Machin</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Luo</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Fong</surname>
              <given-names>KY</given-names>
            </name>
          </person-group>
          <article-title>Development of a smiling touchscreen multimedia program for HRQOL assessment in subjects with varying levels of literacy</article-title>
          <source>Value Health</source>
          <year>2006</year>
          <volume>9</volume>
          <issue>5</issue>
          <fpage>312</fpage>
          <lpage>9</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1098-3015(10)60440-8"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/j.1524-4733.2006.00120.x</pub-id>
          <pub-id pub-id-type="medline">16961549</pub-id>
          <pub-id pub-id-type="pii">S1098-3015(10)60440-8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Thornberry</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>KB</given-names>
            </name>
            <name name-style="western">
              <surname>El-Khorazaty</surname>
              <given-names>MN</given-names>
            </name>
            <name name-style="western">
              <surname>Kiely</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Acceptance, communication mode and use of audio computer-assisted self interview using touchscreen to identify risk factors among pregnant minority women</article-title>
          <source>Methods Rep RTI Press</source>
          <year>2010</year>
          <month>01</month>
          <day>1</day>
          <volume>15</volume>
          <fpage>1001</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/21637729"/>
          </comment>
          <pub-id pub-id-type="doi">10.3768/rtipress.2010.mr.0015.1001</pub-id>
          <pub-id pub-id-type="medline">21637729</pub-id>
          <pub-id pub-id-type="pmcid">PMC3104410</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Saadé</surname>
              <given-names>RG</given-names>
            </name>
            <name name-style="western">
              <surname>Kira</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Computer anxiety in e-learning: the effect of computer self-efficacy</article-title>
          <source>J Info Technol Educ</source>
          <year>2009</year>
          <volume>8</volume>
          <fpage>177</fpage>
          <lpage>91</lpage>
          <pub-id pub-id-type="doi">10.28945/166</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Oliver</surname>
              <given-names>RL</given-names>
            </name>
          </person-group>
          <article-title>A cognitive model of the antecedents and consequences of satisfaction decisions</article-title>
          <source>J Mark Res</source>
          <year>1980</year>
          <month>11</month>
          <volume>17</volume>
          <issue>4</issue>
          <fpage>460</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jstor.org/stable/3150499?origin=crossref"/>
          </comment>
          <pub-id pub-id-type="doi">10.2307/3150499</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Baur</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>An analysis of factors underlying e-health disparities</article-title>
          <source>Camb Q Healthc Ethics</source>
          <year>2008</year>
          <volume>17</volume>
          <issue>4</issue>
          <fpage>417</fpage>
          <lpage>28</lpage>
          <pub-id pub-id-type="doi">10.1017/S0963180108080547</pub-id>
          <pub-id pub-id-type="medline">18724881</pub-id>
          <pub-id pub-id-type="pii">S0963180108080547</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chou</surname>
              <given-names>WS</given-names>
            </name>
            <name name-style="western">
              <surname>Prestin</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Lyons</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Wen</surname>
              <given-names>KY</given-names>
            </name>
          </person-group>
          <article-title>Web 2.0 for health promotion: reviewing the current evidence</article-title>
          <source>Am J Public Health</source>
          <year>2013</year>
          <month>01</month>
          <volume>103</volume>
          <issue>1</issue>
          <fpage>e9</fpage>
          <lpage>18</lpage>
          <pub-id pub-id-type="doi">10.2105/AJPH.2012.301071</pub-id>
          <pub-id pub-id-type="medline">23153164</pub-id>
          <pub-id pub-id-type="pmcid">PMC3518341</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref55">
        <label>55</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bonevski</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Randell</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Paul</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Chapman</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Twyman</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Bryant</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Brozek</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Hughes</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Reaching the hard-to-reach: a systematic review of strategies for improving health and medical research with socially disadvantaged groups</article-title>
          <source>BMC Med Res Methodol</source>
          <year>2014</year>
          <month>03</month>
          <day>25</day>
          <volume>14</volume>
          <fpage>42</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedresmethodol.biomedcentral.com/articles/10.1186/1471-2288-14-42"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1471-2288-14-42</pub-id>
          <pub-id pub-id-type="medline">24669751</pub-id>
          <pub-id pub-id-type="pii">1471-2288-14-42</pub-id>
          <pub-id pub-id-type="pmcid">PMC3974746</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
