<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JFR</journal-id>
      <journal-id journal-id-type="nlm-ta">JMIR Form Res</journal-id>
      <journal-title>JMIR Formative Research</journal-title>
      <issn pub-type="epub">2561-326X</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v6i6e36521</article-id>
      <article-id pub-id-type="pmid">35731563</article-id>
      <article-id pub-id-type="doi">10.2196/36521</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Predictors of Disengagement and Symptom Improvement Among Adults With Depression Enrolled in Talkspace, a Technology-Mediated Psychotherapy Platform: Naturalistic Observational Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Mavragani</surname>
            <given-names>Amaryllis</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Rostam Niakan Kalhori </surname>
            <given-names>Sharareh</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Tannoubi</surname>
            <given-names>Amayra</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Darnell</surname>
            <given-names>Doyanne</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>Department of Psychiatry &#38; Behavioral Sciences</institution>
            <institution>University of Washington</institution>
            <addr-line>325 Ninth Avenue</addr-line>
            <addr-line>Box 359911</addr-line>
            <addr-line>Seattle, WA, 98104</addr-line>
            <country>United States</country>
            <phone>1 206 744 9108</phone>
            <email>darnelld@uw.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-0309-8774</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Pullmann</surname>
            <given-names>Michael D</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8404-864X</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Hull</surname>
            <given-names>Thomas D</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-8275-6765</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Chen</surname>
            <given-names>Shiyu</given-names>
          </name>
          <degrees>MPH</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-2902-0403</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author">
          <name name-style="western">
            <surname>Areán</surname>
            <given-names>Patricia</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-5971-6319</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Psychiatry &#38; Behavioral Sciences</institution>
        <institution>University of Washington</institution>
        <addr-line>Seattle, WA</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Talkspace</institution>
        <addr-line>Seattle, WA</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Doyanne Darnell <email>darnelld@uw.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>6</month>
        <year>2022</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>22</day>
        <month>6</month>
        <year>2022</year>
      </pub-date>
      <volume>6</volume>
      <issue>6</issue>
      <elocation-id>e36521</elocation-id>
      <history>
        <date date-type="received">
          <day>16</day>
          <month>1</month>
          <year>2022</year>
        </date>
        <date date-type="rev-request">
          <day>19</day>
          <month>4</month>
          <year>2022</year>
        </date>
        <date date-type="rev-recd">
          <day>6</day>
          <month>5</month>
          <year>2022</year>
        </date>
        <date date-type="accepted">
          <day>22</day>
          <month>5</month>
          <year>2022</year>
        </date>
      </history>
      <copyright-statement>©Doyanne Darnell, Michael D Pullmann, Thomas D Hull, Shiyu Chen, Patricia Areán. Originally published in JMIR Formative Research (https://formative.jmir.org), 22.06.2022.</copyright-statement>
      <copyright-year>2022</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in JMIR Formative Research, is properly cited. The complete bibliographic information, a link to the original publication on https://formative.jmir.org, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://formative.jmir.org/2022/6/e36521" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Depression is a common psychiatric condition with an estimated lifetime prevalence for major depression of 16.6% in the US adult population and is effectively treated through psychotherapy. The widespread availability of the internet and personal devices such as smartphones are changing the landscape of delivery of psychotherapy; however, little is known about whether and for whom this type of therapy is beneficial, and whether having synchronous video-based sessions provides additional benefits to clients above and beyond messaging-based therapy.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study examined the outcomes associated with the use of a digital platform (Talkspace) for technology-mediated psychotherapy. We examined the duration of client engagement in therapy and client depression score trajectories over 16 weeks. We explored the association of client characteristics, therapist characteristics, and service plan type with time-to-disengagement and trajectories of change in depression scores.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>This naturalistic observational study assessed data collected routinely by the platform between January 2016 and January 2018 and examined psychotherapy outcomes among a large representative sample of adult clients with clinically significant depression. Treatment disengagement was defined as a lack of client-initiated communication for more than 4 weeks. Clients completed the Patient Health Questionnaire-8 item (PHQ-8) at intake and every 3 weeks via an in-app survey. Cox regression analysis was used to examine the time until and predictors of disengagement. Changes in depression scores and predictors of change over time were examined using mixed-effects regression.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>The study included 5890 clients and 1271 therapists. Client scores on the PHQ-8 declined over time, with the average client improving from a score of 15 to below the clinical cutoff of 10 by week 6. At the same time point, 37% of clients had disengaged from the therapy. When combined into a final Cox regression model, those who were more likely to disengage were clients aged 18 to 25 years versus those aged ≥50 years (odds ratio [OR] 0.82, 95% CI 0.74-0.9; <italic>P</italic>&#60;.001), had higher education (OR 1.14, 95% CI 1.06-1.22; <italic>P</italic>&#60;.001), had been in therapy before (OR 1.09, 95% CI 1.02-1.17; <italic>P</italic>=.01), and were living with a partner but unmarried versus single (OR 1.14, 95% CI 1.02-1.27; <italic>P</italic>=.02). Having a therapist with &#62;10 years of experience was related to lower odds of disengagement (OR 0.87, 95% CI 0.8-0.94; <italic>P</italic>=.01). When combined into a final regression model predicting improvement in depression scores over time, clients showing more improvement were those with an associate’s degree or higher (linear estimate=−0.07, <italic>P</italic>=.002) and higher intake PHQ-8 scores (estimate=3.73, <italic>P</italic>&#60;.001). There were no differences based on the plan type.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>Our findings add to the growing literature showing the benefits of technology-mediated psychotherapy over a relatively brief period (16 weeks).</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>depression</kwd>
        <kwd>psychotherapy</kwd>
        <kwd>disengagement</kwd>
        <kwd>internet</kwd>
        <kwd>web-based</kwd>
        <kwd>technology-mediated psychotherapy</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>Depression is a common psychiatric condition with an estimated lifetime prevalence of major depression of 16.6% and a previous year prevalence of a major depressive episode of 7.8% to 8.6% among the US adult population [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref2">2</xref>]. Psychotherapy is an effective treatment for depression [<xref ref-type="bibr" rid="ref3">3</xref>] and is often preferred to medication [<xref ref-type="bibr" rid="ref4">4</xref>]; however, clients face numerous challenges in obtaining what has traditionally been face-to-face in-person treatment, such as the need to travel to a psychotherapist’s office and potential stigma associated with accessing this care [<xref ref-type="bibr" rid="ref5">5</xref>]. The widespread availability of the internet and personal devices such as smartphones is changing the landscape of delivery of psychotherapy, allowing consumers to access care more flexibly and without the need to go to a clinic [<xref ref-type="bibr" rid="ref6">6</xref>]. These opportunities continue to grow in the wake of the COVID-19 pandemic, and the resultant increase in web-based health care is initiated in response to social distancing policies [<xref ref-type="bibr" rid="ref7">7</xref>].</p>
        <p>Numerous technology platforms are now available that allow clients to flexibly communicate with a psychotherapist via text messages, voice, or videoconferencing software; this is known as technology-mediated psychotherapy [<xref ref-type="bibr" rid="ref6">6</xref>]. A common distinction made regarding technology-mediated psychotherapy is whether it is conducted in an asynchronous or synchronous fashion. Asynchronous refers to communication sent and received by therapists and clients at different times, such as a therapist sending a client an email with a homework assignment and then sending feedback on the assignment once completed. Synchronous refers to communication sent and received by therapists and clients at the same time, such as telephone conversations or a web-based chat session.</p>
      </sec>
      <sec>
        <title>Prior Work</title>
        <p>Over the past 20 years, numerous clinical trials of technology-mediated synchronous and asynchronous psychotherapy have been conducted, most of which have evaluated structured cognitive-behavioral approaches. Findings from meta-analyses based on trials conducted in English of psychotherapy for depression by videoconference or telephone (synchronous) indicate that these forms of therapy are similarly effective to face-to-face in-person treatment and with similar (~80%), if not better, completion rates [<xref ref-type="bibr" rid="ref8">8</xref>-<xref ref-type="bibr" rid="ref10">10</xref>]. These meta-analyses reflect diverse samples recruited from a variety of settings (eg, veterans, primary care, HIV clinics, outpatient psychotherapy clinics, and community samples).</p>
        <p>Fewer studies on the effectiveness of text- or chat-based therapy (whether synchronous or asynchronous) have been conducted, and the findings have been mixed. One clinical trial evaluating asynchronous cognitive behavior therapy (CBT)–oriented psychotherapy for depression among adults in Sweden assessed the benefit of an 8-week email-based therapy, indicating that email therapy was associated with pre-post benefit but no difference in treatment outcomes as compared with a guided 8-week self-help web-based program or waitlist-control at 6 months [<xref ref-type="bibr" rid="ref11">11</xref>]. The rates of client completion of the intervention were similarly high between the groups (96% completed the email-based therapy and 93% completed the self-help). Another trial using a web-based platform for chat or text-based psychotherapy found that adult primary care patients in the United Kingdom assigned to a web-based CBT-oriented therapy for 16 weeks experienced greater improvement in depression symptoms than a waitlist or usual primary care comparison group over 8 months [<xref ref-type="bibr" rid="ref12">12</xref>]. Analyses were based on an intent-to-treat sample, of which a large proportion (48%) did not complete the therapy as intended, according to the therapist’s report.</p>
        <p>Few studies on technology-mediated psychotherapy have examined how client demographics moderate outcomes or therapy completion. A secondary exploratory analysis of the United Kingdom chat or text-based CBT trial [<xref ref-type="bibr" rid="ref12">12</xref>] examined moderators of treatment outcome at the 4-month follow-up and found that depression outcomes were better for patients with higher baseline depression severity [<xref ref-type="bibr" rid="ref13">13</xref>]. In addition, being separated, divorced, or widowed as opposed to being single or married was associated with greater treatment outcomes for those in the CBT condition; age and education level were not significant moderators. Findings from a meta-analysis of a broader array of technology-mediated psychological interventions targeting various mental health diagnoses suggest that client age may be a relevant moderator of outcome; in particular, adults aged 19 to 39 years may experience greater benefits than older or younger clients [<xref ref-type="bibr" rid="ref14">14</xref>].</p>
        <p>Little is known about the effects of technology-mediated psychotherapy delivered routinely, for whom this approach may be beneficial, or whether there is differential effectiveness based on asynchronous versus synchronous delivery. In addition, to date, no studies have compared asynchronous and synchronous technology-mediated psychotherapy. This study aims to fill these gaps by studying outcomes associated with the use of a digital platform (Talkspace) that facilitates technology-mediated psychotherapy either asynchronously or synchronously among adult clients experiencing depression symptoms. Specifically, the platform is a desktop computer and mobile smartphone app that allows 24/7 communication via a secure interactive text chat and voice or video messages with a licensed psychotherapist for a fee [<xref ref-type="bibr" rid="ref15">15</xref>]. Clients may pay for a plan with optional synchronous 1-hour sessions conducted through chat, telephone, or videoconferencing that can be either 1 or 4 times per month. Several observational studies with selected samples of Talkspace users suggest that the use of the asynchronous text-based psychotherapy plan is associated with improvement in depression symptoms over 12 to 16 weeks of treatment; however, none of these studies examined predictors of these outcomes and for whom the therapy may be most beneficial [<xref ref-type="bibr" rid="ref15">15</xref>-<xref ref-type="bibr" rid="ref18">18</xref>].</p>
      </sec>
      <sec>
        <title>Objective</title>
        <p>This naturalistic observational study harnesses data collected routinely by the platform and examines psychotherapy outcomes among a large representative sample of clients with clinically significant depression. We examined how long clients remained in therapy before disengaging from the application after their initial enrollment as well as client trajectories of depression scores over 16 weeks after enrollment. In addition, we explored the association between client characteristics, therapist characteristics, and the type of service plan with time-to-disengagement and trajectories of change in depression scores.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Design and Procedure</title>
        <p>This study included longitudinal and observational research with clients who signed up for Talkspace between January 2016 and January 2018 and their therapists using data collected routinely as part of the service. Clients accessed the service through an internet search and either paid out-of-pocket or submitted costs to a private insurance plan for out-of-network reimbursement. The signup process includes a brief intake with a consultation therapist who helps the client select their desired plan (eg, messaging only, monthly video, or weekly video plans) and records the client’s therapist preferences, presenting complaints, and demographics. This information is used by the service to identify and offer the client a choice among 3 therapists that most closely match the client’s preferences, are licensed in the client’s state of residence, and have a successful history of treating conditions the client is reporting. After therapist selection, the client is offered a baseline symptom assessment and can then message their therapist as often as they like. Therapists respond to client messages within 24 hours, gather information for a diagnosis, explain the frame of the medium, and conduct informed consent procedures, after which therapy proceeds 5 days a week.</p>
        <p>Therapists encounter the platform through internet searches, professional organizations, and peer contacts. All prospective therapists completed an application process to verify the state licensure, training and degree type, professional insurance, background checks, and confirmation of meeting the National Certification of Quality Assurance standards. Following verification, therapists completed an orientation to the platform, received Health Insurance Portability and Accountability Act, privacy, and security training on the proper use of technological media for delivering care, and took on a small number of training cases with supervision for 30 days before beginning their practice on the platform. Engagement and outcome metrics were monitored after the training phase for quality assurance along with a random peer-review process by other therapists on the platform. The service does not prescribe a specific approach to therapy, and while most therapists (61%) report practicing from a cognitive-behavioral orientation, therapists from many traditions are represented on the platform.</p>
      </sec>
      <sec>
        <title>Participants</title>
        <p>The participants in this study were clients of the Talkspace service. Inclusion criteria was age 18 years and older, with clinically significant depression as indicated by having a primary diagnosis of depression or a depression-related disorder, and a score of 10 or greater on the 8-item version of the Patient Health Questionnaire (PHQ-8) administered at intake for the service. In addition, given the requirements of the service, participants were English language literate, had internet access, and were able to use mobile or desktop applications for the service. Exclusion criteria for the use of the service were indications during the intake with the consultation therapist, or at any point when working with the treating therapist, of any schizophrenia spectrum and psychotic disorder, or any diagnosis with psychotic features, or any condition requiring hospitalization, or suicidal thoughts or behavior sufficient to be marked a <italic>Yes</italic> on any of questions 3 through 6 on the Columbia Suicide Severity Rating Scale Lifetime-Recent Screen [<xref ref-type="bibr" rid="ref19">19</xref>] that would require a more intensive level of care than can be offered through an outpatient service.</p>
      </sec>
      <sec>
        <title>Measures and Variables</title>
        <sec>
          <title>Client Characteristics and Demographics</title>
          <p>As part of signing up for the Talkspace service, clients provide their age range (19-25, 26-35, 36-49, 50+ years), gender (female, male, gender queer, transgender male, transgender female, gender variant, other), education level (high school, bachelor’s degree or higher), whether they had ever been in therapy before, and their marital status (single, married, living with a partner, divorced, separated, and widowed).</p>
        </sec>
        <sec>
          <title>Depression-Related Diagnosis</title>
          <p>Clients were assigned a psychiatric diagnosis according to the 5th edition of the Diagnostic and Statistical Manual of Mental Disorders by their matched therapist, which may be assessed through a mix of diagnostic interviewing, often by video, and delivery of standardized symptom assessments. Clients were included in the study if they had a primary diagnosis of a depression-related disorder (eg, major depressive disorder, adjustment disorder with depressed mood, dysthymic disorder, or other mood disorders).</p>
        </sec>
        <sec>
          <title>Therapist Characteristics</title>
          <p>Therapists provide information about demographics and professional experiences when they apply to join a service network. Therapists can select female, male, gender queer, transgender male, transgender female, gender variant, or other gender. Therapists indicate years of postlicensure experience as psychotherapists (coded as &#60;5 years, 5-10 years, and &#62;10 years) and areas of expertise (for this study, dichotomously coded as whether therapists indicated expertise in treating depression).</p>
        </sec>
        <sec>
          <title>Psychotherapy Disengagement</title>
          <p>Disengagement from psychotherapy was defined for this study by a lack of client-initiated communication via any means using the Talkspace application (eg, text, voice messages, and videoconferencing sessions) for more than 4 weeks after initial enrollment. These data are drawn from administrative records on exchanges with therapists and are passively collected by the application.</p>
        </sec>
        <sec>
          <title>Depression Symptoms</title>
          <p>Clients were asked to complete the PHQ-8 [<xref ref-type="bibr" rid="ref20">20</xref>] at intake and every 3 weeks via an in-app survey. There was an allowance for completing these with a 1-week buffer before or after the deadline. The PHQ-8 includes 8 items that assess the severity of depression symptoms according to the Diagnostic Statistical Manual, Fourth Edition, criteria, with the exception of suicidality or preoccupation with death. Items were rated on a scale from 0 (not at all) to 3 (nearly every day) and summed to obtain a total score. Higher scores indicate greater severity, and a cutoff score of 10 or higher indicates clinically significant depression. A 5-point difference in scores indicates a clinically meaningful change in depression symptoms [<xref ref-type="bibr" rid="ref21">21</xref>].</p>
        </sec>
        <sec>
          <title>Plan Type</title>
          <p>Platform clients could opt to purchase one of three service plans: (1) unlimited text, voice, or video messages; (2) unlimited text, voice, or video messages plus once per month 1-hour videoconferencing session; or (3) unlimited text, voice, or video messages plus 4 times per month 1-hour videoconferencing sessions. How often and to what extent clients and therapists communicate varies and is dependent upon the frequency with which clients send messages to their therapist. Therapists are expected to respond to client communications within 24 hours and generally respond within 12 hours, except for therapist days off or other times mutually agreed upon by the therapist and client.</p>
        </sec>
      </sec>
      <sec>
        <title>Plan of Analysis</title>
        <p>Using R, we examined descriptive statistics for client and therapist characteristics, depression symptoms at intake or baseline, and service plan type that clients were enrolled in. We examined the missingness of study variables at each point of time as described in the description of our results. For the prediction analysis, we collapsed several categorical variables when the sample size within some of the variable categories was low, which made theoretical sense to do so. We collapsed education into high school education versus associate’s degree or higher. As only 30 participants were widowed, we collapsed this group into the missing category. We conducted 2 sets of mixed effects regression analyses to examine, (1) trajectories of disengagement over 52 weeks and predictors of disengagement and (2) trajectories of depression scores over 16 weeks and predictors of change in depression over time.</p>
        <sec>
          <title>Predicting Psychotherapy Disengagement Over 52 Weeks After Intake</title>
          <p>We examined how long clients were in therapy before disengaging over the course of 52 weeks, following their intake with the platform. Furthermore, a duration of 52 weeks was selected based on visual inspection of the data, showing that most participants had stopped communicating with their therapist for more than 4 weeks by that time. Using R, we conducted mixed-effects Cox regression analyses predicting time until disengagement, with random terms for clinician and client to account for nesting of time within client and client within therapist. Model building followed appropriate procedures for our combined confirmatory and exploratory analyses; predictors were first tested individually to conduct a priori confirmatory hypothesis testing, and those that were significant at <italic>P</italic>&#60;.05 were included in a final, combined exploratory model to examine the overall model structure. Significance was tested using the log-rank test for individual variable models. For categorical variables, simple contrast comparisons were used in which each category was compared against the reference category, as described in the results. To estimate the goodness-of-fit, we computed the proportion of the variance accounted for using Nagelkerke <italic>R</italic><sup>2</sup> for each model. We also computed the concordance statistic, which provides the fraction of concordant pairs between the model-predicted and actual data and is equal to the receiver operating characteristic curve. A concordance statistic of 0.50 means that the model is no better than random chance, while a statistic of 1.0 would mean there is a perfect concordance between predicted and actual data. Analyses of the relationship between missing data status on demographic and descriptive variables and time until engagement were conducted separately using Cox regression analyses; cases with missing data were excluded from the primary analyses.</p>
        </sec>
        <sec>
          <title>Predicting Depression Symptoms Over 16 Weeks After Intake</title>
          <p>We examined depression symptoms based on PHQ-8 scores over the course of 16 weeks following the client’s service intake. In addition, a duration of 16 weeks was selected based on visual inspection of the data showing that most of the change in scores occurred during the first 16 weeks and missing data were substantial at that point, which is likely primarily owing to high rates of client disengagement from therapy by 16 weeks. Using R, mixed-effects regressions with random terms for therapist and client were computed to adjust standard errors owing to nesting of time within client and client within therapist. The covariance matrix was specified to be unstructured. As described below, the rates of missing data were high for the dependent variable owing to ending treatment or incomplete measures. Missing data were determined to be not missing at random, based on the empirical analyses described below and theoretical reasons. It is highly likely that those who did not complete measures were due to unmeasured reasons, such as treatment attitudes, motivation, and emotional distress or wellness. Therefore, full information maximum likelihood estimation was used for model testing and parameter estimation (to maximize all available data), and variables associated with missing outcome data were included in the final models to statistically adjust for missingness to the greatest extent possible. Findings may only be inferred to the population of clients who completed measures while receiving treatment. Model building followed standard procedures [<xref ref-type="bibr" rid="ref22">22</xref>]. Model fit was tested by computing the Akaike information criterion, Bayesian information criterion, and 2-Log Likelihood deviance statistics. The models tested linear, quadratic, and cubic time trends for changes in the PHQ-8 scores over time. The best-fitting models included the linear and quadratic time trends. Then, each potential predictor, including client and therapist characteristics, was entered into individual models, and interaction terms for linear and quadratic time were tested using model fit deviance statistics, with each more complex model tested against simpler models, and all available data were used for each computation. All variables and interaction terms from models that were statistically significant at <italic>P</italic>&#60;.05 were then entered into a final omnibus model. We ran two types of <italic>R</italic><sup>2</sup> for each model: the conditional <italic>R</italic><sup>2</sup> provides the total proportion of variance accounted for, including fixed and random terms, while the marginal <italic>R</italic><sup>2</sup> provides the proportion of variance accounted for by fixed terms only.</p>
        </sec>
      </sec>
      <sec>
        <title>Ethics Approval</title>
        <p>As part of the terms of service agreement, clients agreed that their anonymized data may be used for research purposes. Considering the data were completely anonymized, this study was exempted from institutional review board approval by the institutional review board of the University of Washington.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Sample Characteristics</title>
        <p>The study included 5890 clients and 1271 therapists (<xref ref-type="table" rid="table1">Table 1</xref>). The participants were primarily female (4504/5890, 76.5%), 26 to 35 years of age (3061/5890, 52%), had a bachelor’s degree (3706/5890, 62.9%), were single (3122/5890, 52.8%), and had been in therapy before (4004/5890, 68%). The average PHQ-8 score was 15.2 (SD 3.9). The therapists were primarily female (1114/1271, 87.6%), with more than 5 years of experience (1017/1271, 80%), and close to half endorsed having depression-specific expertise (591/1271, 46.5%). Clients predominantly signed up for a messaging-only Talkspace plan (5389/1271, 91.5%). There were no missing data for client disengagement from therapy; however, we observed modest amounts of missing data for other client and therapist variables (ranging from 0.3% to 13.6%, see <xref ref-type="table" rid="table1">Table 1</xref>). Missing data analyses found that missing data on the following variables were not significantly associated with the length of time until disengagement: client’s gender, client’s first time in therapy, or therapist’s years of experience. Client age and education were significantly associated, such that those who had missing data on these variables were more likely to disengage (age odds ratio [OR] 1.46, 95% CI 1.15-1.89, <italic>P</italic>=.002; education OR 1.10, 95% CI 1.02-1.19, <italic>P</italic>=.013).</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Client and therapist characteristics.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="620"/>
            <col width="350"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Variable</td>
                <td>Value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="2">Client: total, n (%)</td>
                <td>5890 (100)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Client age (years), n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>18-25</td>
                <td>1328 (22.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td>3061 (52)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td>1182 (20.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td>250 (4.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing</td>
                <td>69 (1.2)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Client gender, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Female</td>
                <td>4505 (76.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>5 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Queer</td>
                <td>9 (0.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Variant</td>
                <td>5 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Male</td>
                <td>1317 (22.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Transgender female</td>
                <td>3 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Transgender male</td>
                <td>7 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing</td>
                <td>39 (0.7)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Client education, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>High school</td>
                <td>1327 (22.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Some college</td>
                <td>14 (0.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Associate’s degree</td>
                <td>6 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Bachelor’s degree or higher</td>
                <td>3706 (62.9)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Master’s degree</td>
                <td>21 (0.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Doctoral degree</td>
                <td>5 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Professional degree</td>
                <td>8 (0.1)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing</td>
                <td>803 (13.6)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Client marital status, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Divorced</td>
                <td>328 (5.6)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Living with a partner</td>
                <td>497 (8.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Married</td>
                <td>1663 (28.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Separated</td>
                <td>142 (2.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Single</td>
                <td>3112 (52.8)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Widowed</td>
                <td>30 (0.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing</td>
                <td>118 (2.0)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Client first time in therapy</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>No, n (%)</td>
                <td>4004 (68)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Yes, n (%)</td>
                <td>1346 (22.9)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing, n (%)</td>
                <td>540 (9.2)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Baseline PHQ-8<sup>a</sup>, mean (SD)</td>
                <td>15.2 (3.9)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Talkspace plan type</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Text only, n (%)</td>
                <td>5389 (91.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>1-month video, n (%)</td>
                <td>411 (7)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>4-month video, n (%)</td>
                <td>90 (1.5)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Therapist, total (N)</td>
                <td>1271 (100)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Therapist gender, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Male</td>
                <td>157 (12.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Female</td>
                <td>1114 (87.6)</td>
              </tr>
              <tr valign="top">
                <td colspan="3">
                  <bold>Therapist years of experience, n (%)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#60;5</td>
                <td>250 (19.7)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>5-10</td>
                <td>551 (43.4)</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Missing</td>
                <td>4 (0.3)</td>
              </tr>
              <tr valign="top">
                <td colspan="2">Therapist with depression expertise, n (%)</td>
                <td>591 (46.5)</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>PHQ-8: Patient Health Questionnaire-8 item.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Predicting Psychotherapy Disengagement Over 52 Weeks Post Intake</title>
        <p>By week 6, 37% of the clients had disengaged from therapy. Half of the sample disengaged from therapy by week 9 and by week 52, nearly all clients had disengaged (n=5441, 92.4%; <xref rid="figure1" ref-type="fig">Figure 1</xref>). <xref ref-type="table" rid="table2">Table 2</xref> displays the single-variable predictor models for all the variables that were significantly associated with disengagement. When combined into a final Cox regression model, significant variables included client age, education, whether the client had been in therapy before, marital status, and years of experience as a therapist (<xref ref-type="table" rid="table3">Table 3</xref>). Compared with those aged 18 to 25 years, those aged 36 to 49 years had 18.9% lower odds of disengaging at any point in time (OR 0.82, 95% CI 0.74-0.9; <italic>P</italic>&#60;.001; <xref rid="figure2" ref-type="fig">Figure 2</xref>) and those aged &#62;50 years had 30.1% lower odds of disengaging (OR 0.70, 95% CI 0.59-0.83; <italic>P</italic>&#60;.001). Those with higher education had 13.5% greater odds of disengaging than those with lower education (OR 1.14, 95% CI 1.06-1.22; <italic>P</italic>&#60;.001), and those who were in therapy for the first time had 8.9% greater odds of disengaging (OR 1.09, 95% CI 1.02-1.17; <italic>P</italic>=.01). Participants who were living with a partner had 13.9% greater odds of disengaging as compared with those who were single (OR 1.14, 95% CI 1.02-1.27; <italic>P</italic>=.02); there were no significant differences between those who were single and those who were married, separated, or divorced. Finally, clients with therapists who had more than 10 years of experience had 13.2% lower odds of disengaging (OR 0.87, 95% CI 0.8-0.94; <italic>P</italic>=.01). Although significant in the single-variable predictor model, there were no significant differences in the likelihood of disengagement for clients based on the therapist endorsement of depression-specific expertise in the combined Cox regression model. Plan type, intake PHQ-8 score, client gender, and therapist gender were not included in the final model, as these variables were not significant in the single-variable predictor models. Model fit statistics revealed a very low proportion of variance accounted for and concordance across all models (<xref ref-type="table" rid="table3">Tables 3</xref> and <xref ref-type="table" rid="table4">4</xref>). The proportion of variance accounted for in the final model was only 1.8%, and the concordance statistic of 0.54 indicated that the model predicted the length of time until disengagement only slightly better than chance.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Time until disengagement.</p>
          </caption>
          <graphic xlink:href="formative_v6i6e36521_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Significant single-variable predictor Cox regression models testing time until treatment disengagement<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="300"/>
            <col width="170"/>
            <col width="170"/>
            <col width="110"/>
            <col width="0"/>
            <col width="110"/>
            <col width="0"/>
            <col width="110"/>
            <thead>
              <tr valign="top">
                <td colspan="2">
                  <break/>
                </td>
                <td>Median (weeks)</td>
                <td>OR<sup>b</sup> (95% CI)</td>
                <td><italic>P</italic> value</td>
                <td colspan="2">Log rank</td>
                <td colspan="2"><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="6">
                  <bold>Age (years)</bold>
                </td>
                <td colspan="2">33.78</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>18-25</td>
                <td>8</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td>9</td>
                <td>0.93 (0.87-0.99)</td>
                <td>.035</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td>10</td>
                <td>0.83 (0.77-0.91)</td>
                <td>&#60;.001</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td>11</td>
                <td>0.71 (0.61-0.82)</td>
                <td>&#60;.001</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>Education</bold>
                </td>
                <td colspan="2">17.24</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>High school</td>
                <td>9</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Associate’s degree or higher</td>
                <td>8</td>
                <td>1.15 (1.08-1.22)</td>
                <td>&#60;.001</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>First time in treatment</bold>
                </td>
                <td colspan="2">11.37</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>No</td>
                <td>9</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Yes</td>
                <td>8</td>
                <td>1.12 (1.05-1.19)</td>
                <td>&#60;.001</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>Marital status</bold>
                </td>
                <td colspan="2">15.31</td>
                <td>.004</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Single</td>
                <td>9</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Living with partner</td>
                <td>8</td>
                <td>1.18 (1.07-1.3)</td>
                <td>.001</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Married</td>
                <td>9</td>
                <td>0.96 (0.91-1.03)</td>
                <td>.22</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Separated</td>
                <td>10</td>
                <td>0.98 (0.82-1.17)</td>
                <td>.83</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Divorced</td>
                <td>9</td>
                <td>0.95 (0.84-1.06)</td>
                <td>.37</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>Therapist experience</bold>
                </td>
                <td colspan="2">16.45</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#60;5 years</td>
                <td>8</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>5-10 years</td>
                <td>9</td>
                <td>0.93 (0.86-1)</td>
                <td>.07</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;10 years</td>
                <td>9</td>
                <td>0.86 (0.8-0.93)</td>
                <td>.002</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="6">
                  <bold>Therapist with depression-specific expertise</bold>
                </td>
                <td colspan="2">7.97</td>
                <td>.004</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Not endorsed</td>
                <td>8</td>
                <td>Ref</td>
                <td>Ref</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Endorsed</td>
                <td>9</td>
                <td>0.92 (0.88-0.98)</td>
                <td>.03</td>
                <td colspan="2">
                  <break/>
                </td>
                <td colspan="2">
                  <break/>
                </td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table2fn1">
              <p><sup>a</sup>Client n=5890, therapist n=1271. Variables that were tested and were not significant included plan type, intake PHQ-8 score, client gender, and therapist gender. Widowed data were collapsed with missing marital status data. Disengagement from treatment was defined as &#62;4 weeks without any client-initiated communication to the therapist, such as texting, video exchanges, images, or audio clips. The modeling included up to 52 weeks of therapy.</p>
            </fn>
            <fn id="table2fn2">
              <p><sup>b</sup>OR: odds ratio.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table3">
          <label>Table 3</label>
          <caption>
            <p>Model fit statistics for individual and final combined nested Cox regression model predicting time until treatment disengagement<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="370"/>
            <col width="0"/>
            <col width="300"/>
            <col width="300"/>
            <thead>
              <tr valign="top">
                <td colspan="3">
                  <break/>
                </td>
                <td>Nagelkerke <italic>R</italic><sup>2</sup></td>
                <td>Concordance</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="5">
                  <bold>Individual models</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Age</td>
                <td colspan="2">0.0065</td>
                <td>0.52</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Education</td>
                <td colspan="2">0.0036</td>
                <td>0.52</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in therapy</td>
                <td colspan="2">0.0022</td>
                <td>0.51</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Marital status</td>
                <td colspan="2">0.0028</td>
                <td>0.51</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Therapist experience</td>
                <td colspan="2">0.0030</td>
                <td>0.51</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Therapist with depression-specific expertise</td>
                <td colspan="2">0.0015</td>
                <td>0.51</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Final model</td>
                <td>0.018</td>
                <td>0.54</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table3fn1">
              <p><sup>a</sup>Disengagement from treatment was defined as &#62;4 weeks without any client-initiated communication to the therapist, such as texting, video exchanges, images, or audio clips. The modeling included up to 52 weeks of therapy.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Time until disengagement stratified by age group.</p>
          </caption>
          <graphic xlink:href="formative_v6i6e36521_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <table-wrap position="float" id="table4">
          <label>Table 4</label>
          <caption>
            <p>Final combined nested Cox regression model predicting time until treatment disengagement<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="370"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="120"/>
            <thead>
              <tr valign="top">
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">Coefficient</td>
                <td colspan="2">OR<sup>b</sup> (95% CI)</td>
                <td><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="8">
                  <bold>Age (years; ref=18-25)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td colspan="2">−0.05</td>
                <td colspan="2">0.96 (0.88-1.03)</td>
                <td colspan="2">.27</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td colspan="2">−0.20</td>
                <td colspan="2">0.82 (0.74-0.9)</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td colspan="2">−0.36</td>
                <td colspan="2">0.7 (0.59-0.83)</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Education (ref=high school)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Associate’s or bachelor’s degree or higher</td>
                <td colspan="2">0.13</td>
                <td colspan="2">1.14 (1.06-1.22)</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in treatment</td>
                <td colspan="2">0.09</td>
                <td colspan="2">1.09 (1.02-1.17)</td>
                <td colspan="2">.011</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Marital status (ref=single)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Living with a partner</td>
                <td colspan="2">0.13</td>
                <td colspan="2">1.14 (1.02-1.27)</td>
                <td colspan="2">.019</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Married</td>
                <td colspan="2">0.03</td>
                <td colspan="2">1.03 (0.95-1.1)</td>
                <td colspan="2">.48</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Separated</td>
                <td colspan="2">0.006</td>
                <td colspan="2">1.01 (0.82-1.23)</td>
                <td colspan="2">.95</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Divorced</td>
                <td colspan="2">0.04</td>
                <td colspan="2">1.04 (0.91-1.2)</td>
                <td colspan="2">.55</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Therapist experience (years; ref=&#60;5)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>5-10</td>
                <td colspan="2">−0.09</td>
                <td colspan="2">0.92 (0.84-0.99)</td>
                <td colspan="2">.06</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;10</td>
                <td colspan="2">−0.14</td>
                <td colspan="2">0.87 (0.8-0.94)</td>
                <td colspan="2">.01</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Therapist with depression-specific expertise</td>
                <td colspan="2">−0.06</td>
                <td colspan="2">0.94 (0.89-1)</td>
                <td colspan="2">.14</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table4fn1">
              <p><sup>a</sup>Disengagement from treatment was defined as &#62;4 weeks without any client-initiated communication to the therapist, such as texting, video exchanges, images, or audio clips. The modeling included up to 52 weeks of therapy. Predictor variables that were significant in simpler models were also included.</p>
            </fn>
            <fn id="table4fn2">
              <p><sup>b</sup>OR: odds ratio.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
      </sec>
      <sec>
        <title>Predicting Depression Symptoms Over 16 Weeks Post Intake</title>
        <p>There were high levels of missingness in the PHQ-8 over the 16 weeks after the intake observation period. Of the 5890 clients, 3447 (58.5%) completed a 3-week survey, 1978 (33.6%) completed a 6-week survey, 1165 (19.8%) completed a 9-week survey, 724 (12.3%) completed a 12-week survey, and 506 (8.6%) completed a 15-week survey. When comparing those with at least one follow-up time point to those without, there were no differences in the rates of missing data on the basis of age, education level, history of therapy, gender, or PHQ-8 score. Those who did not have any follow-up PHQ-8 were more likely to be divorced (8.1% with follow-up data vs 9.6% missing all follow-up) or living with a partner (8.1% vs 9.6%), less likely to be married (30.2% vs 27.1%), more likely to have a video plan (7.9% vs 9.4%), and less likely to have a messaging-only plan (90.6% vs 92.1%).</p>
        <p>The best-fitting model for change over time in the PHQ-8 is shown in <xref rid="figure3" ref-type="fig">Figure 3</xref>, illustrating a curvilinear change in PHQ-8 scores during the first 16 weeks of treatment (quadratic -2LL<sub>5</sub>=−41,432, deviance=399, <italic>P</italic>&#60;.001), therefore all subsequent models included linear and quadratic change variables. The model-derived average PHQ-8 score at intake was 14.94 (SE 0.08), with a linear slope decreasing an average of −1.25 points (SE 0.03) between the first and second week and flattening by an additional 0.06 points (SE 0.002) for each subsequent week. Client scores on the PHQ-8 declined over time, dropping below the clinical cut-off from a score of 15 to 10 by week 6.</p>
        <p><xref ref-type="table" rid="table5">Table 5</xref> displays models using single variables to predict change over time and found that client education level and age significantly predicted intake PHQ-8 score and linear change over time (education level linear – 2LL<sub>2</sub> deviance=25, <italic>P</italic>&#60;.001; age linear 2LL<sub>6</sub> deviance=9, <italic>P</italic>=.006). Clients’ PHQ-8 score at intake, and whether it was their first time in therapy, significantly predicted linear and quadratic change over time in the PHQ-8 score (first PHQ-8 score linear -2LL<sub>2</sub> deviance=1991, <italic>P</italic>&#60;.001, quadratic -2LL<sub>1</sub> deviance=99, <italic>P</italic>&#60;.001; first time in therapy linear -2LL<sub>2</sub> deviance=12, <italic>P</italic>&#60;.001, quadratic -2LL<sub>1</sub> deviance=9, <italic>P</italic>&#60;.001). Variables that were not significantly associated with PHQ-8 scores over time included plan type, client gender, therapist gender, therapist years of experience, and therapist endorsing depression-specific expertise. Model fit statistics are displayed in <xref ref-type="table" rid="table6">Table 6</xref>, demonstrating that, unsurprisingly, the model incorporating intake PHQ-8 score accounted for the largest amount of variance in the model.</p>
        <p>The final model explained 51.1% of the variance in PHQ-8 score change. This model included predictor variables that had significant relationships with the intercept and linear or quadratic change in PHQ-8 scores over time, on the basis of single variable modeling (see <xref ref-type="table" rid="table7">Table 7</xref>). Significant predictors were as follows: those with an associate’s degree or higher improved more quickly based on their sharper linear slope (linear estimate=−0.07, <italic>P</italic>=.002). Higher intake PHQ-8 scores, displayed as a median split in <xref rid="figure4" ref-type="fig">Figure 4</xref>, were associated with a larger PHQ-8 intercept (estimate=3.73, <italic>P</italic>&#60;.001), faster linear improvement (estimate=−0.50, <italic>P</italic>&#60;.001) and faster flattening (quadratic estimate=0.03, <italic>P</italic>&#60;.001). There were no differences in the PHQ-8 intercept on whether it was the client’s first time in therapy; however, first-time clients had faster linear improvement (linear estimate=−0.37, <italic>P</italic>&#60;.001) and faster flattening (quadratic estimate=0.03, <italic>P</italic>&#60;.001). Age was not significant after controlling for other variables.</p>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>Model estimated change on the Patient Health Questionnaire-8 item (PHQ-8) over 16 weeks with SE shading.</p>
          </caption>
          <graphic xlink:href="formative_v6i6e36521_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <table-wrap position="float" id="table5">
          <label>Table 5</label>
          <caption>
            <p>Significant single-variable mixed effects models predicting Patient Health Questionnaire-8 item (PHQ-8) score change<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="370"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="120"/>
            <thead>
              <tr valign="top">
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">Estimate</td>
                <td colspan="2">SE</td>
                <td><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="8">
                  <bold>Education (ref=high school)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Intercept</td>
                <td colspan="2">15.66</td>
                <td colspan="2">0.13</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week</td>
                <td colspan="2">−1.22</td>
                <td colspan="2">0.03</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week<sup>2</sup></td>
                <td colspan="2">0.06</td>
                <td colspan="2">0.002</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Associate’s or bachelor’s degree or higher</td>
                <td colspan="2">−0.90</td>
                <td colspan="2">0.15</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Associate’s degree or higher × week</td>
                <td colspan="2">−0.02</td>
                <td colspan="2">0.02</td>
                <td colspan="2">.34</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Age (ref=18-25 years)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Intercept</td>
                <td colspan="2">15.26</td>
                <td colspan="2">0.13</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week</td>
                <td colspan="2">−1.24</td>
                <td colspan="2">0.03</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week<sup>2</sup></td>
                <td colspan="2">0.06</td>
                <td colspan="2">0.002</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td colspan="2">−0.49</td>
                <td colspan="2">0.14</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td colspan="2">−0.14</td>
                <td colspan="2">0.17</td>
                <td colspan="2">.41</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td colspan="2">−0.42</td>
                <td colspan="2">0.29</td>
                <td colspan="2">.14</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35 × week</td>
                <td colspan="2">−0.01</td>
                <td colspan="2">0.03</td>
                <td colspan="2">.67</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49 × week</td>
                <td colspan="2">0.005</td>
                <td colspan="2">0.03</td>
                <td colspan="2">.87</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50 × week</td>
                <td colspan="2">0.05</td>
                <td colspan="2">0.05</td>
                <td colspan="2">.27</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Intake PHQ-8 score</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Intercept</td>
                <td colspan="2">15.07</td>
                <td colspan="2">0.06</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week</td>
                <td colspan="2">−1.28</td>
                <td colspan="2">0.02</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week<sup>2</sup></td>
                <td colspan="2">0.07</td>
                <td colspan="2">0.002</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>PHQ-8</td>
                <td colspan="2">3.75</td>
                <td colspan="2">0.05</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>PHQ-8 × week</td>
                <td colspan="2">−0.49</td>
                <td colspan="2">0.02</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>PHQ-8 × week<sup>2</sup></td>
                <td colspan="2">0.03</td>
                <td colspan="2">0.002</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>First time in therapy</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Intercept</td>
                <td colspan="2">15.04</td>
                <td colspan="2">0.09</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week</td>
                <td colspan="2">−1.17</td>
                <td colspan="2">0.03</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Week <sup>2</sup></td>
                <td colspan="2">0.06</td>
                <td colspan="2">0.003</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in therapy</td>
                <td colspan="2">−0.20</td>
                <td colspan="2">0.15</td>
                <td colspan="2">.17</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in therapy × week</td>
                <td colspan="2">−0.30</td>
                <td colspan="2">0.07</td>
                <td colspan="2">&#60;.001</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in therapy × week<sup>2</sup></td>
                <td colspan="2">0.02</td>
                <td colspan="2">0.005</td>
                <td colspan="2">&#60;.001</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table5fn1">
              <p><sup>a</sup>Variables that were tested and were not significant on the basis of model fit statistics included client gender, marital status, plan type, therapist gender, therapist years of experience, and therapist endorsing depression-specific expertise.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table6">
          <label>Table 6</label>
          <caption>
            <p>Model fit statistics predicting Patient Health Questionnaire-8 item (PHQ-8) score change.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="230"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="90"/>
            <col width="0"/>
            <col width="130"/>
            <col width="0"/>
            <col width="130"/>
            <thead>
              <tr valign="top">
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">AIC<sup>a</sup></td>
                <td colspan="2">BIC<sup>b</sup></td>
                <td colspan="2">−2LL<sup>c</sup></td>
                <td colspan="2">df</td>
                <td colspan="2">Conditional <italic>R</italic><sup>2</sup></td>
                <td>Marginal <italic>R</italic><sup>2</sup></td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="14">
                  <bold>Individual models</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Education</td>
                <td colspan="2">66,153.2</td>
                <td colspan="2">66,204.5</td>
                <td colspan="2">−33,069.6</td>
                <td colspan="2">7</td>
                <td colspan="2">0.279</td>
                <td colspan="2">0.180</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Age</td>
                <td colspan="2">66,082.0</td>
                <td colspan="2">66,162.7</td>
                <td colspan="2">−33,030</td>
                <td colspan="2">11</td>
                <td colspan="2">0.271</td>
                <td colspan="2">0.179</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Intake PHQ-8 score</td>
                <td colspan="2">62,024.4</td>
                <td colspan="2">62,083.0</td>
                <td colspan="2">−31,004.2</td>
                <td colspan="2">8</td>
                <td colspan="2">0.510</td>
                <td colspan="2">0.460</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>First time in therapy</td>
                <td colspan="2">62,877.0</td>
                <td colspan="2">62,935.3</td>
                <td colspan="2">−31,430.5</td>
                <td colspan="2">8</td>
                <td colspan="2">0.278</td>
                <td colspan="2">0.182</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Final combined model</td>
                <td colspan="2">64,112.8</td>
                <td colspan="2">64,252.3</td>
                <td colspan="2">−32,037.4</td>
                <td colspan="2">19</td>
                <td colspan="2">0.511</td>
                <td>0.455</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table6fn1">
              <p><sup>a</sup>AIC: Akaike Information Criterion.</p>
            </fn>
            <fn id="table6fn2">
              <p><sup>b</sup>BIC: Bayesian Information Criterion.</p>
            </fn>
            <fn id="table6fn3">
              <p><sup>c</sup>−2LL: −2 Log Likelihood.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table7">
          <label>Table 7</label>
          <caption>
            <p>Final combined mixed effects model predicting Patient Health Questionnaire (PHQ-8) score change<sup>a</sup>.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="370"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="240"/>
            <col width="0"/>
            <col width="120"/>
            <thead>
              <tr valign="top">
                <td colspan="3">
                  <break/>
                </td>
                <td colspan="2">Estimate</td>
                <td colspan="2">SE</td>
                <td><italic>P</italic> value</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="3">Intercept</td>
                <td colspan="2">15.36</td>
                <td colspan="2">0.14</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Week</td>
                <td colspan="2">−1.10</td>
                <td colspan="2">0.04</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Week<sup>2</sup></td>
                <td colspan="2">0.06</td>
                <td colspan="2">0.002</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Associate’s or bachelor’s degree or higher (ref=high school)</td>
                <td colspan="2">−0.18</td>
                <td colspan="2">0.12</td>
                <td>.120</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Associate’s degree or higher × week (ref=high school)</td>
                <td colspan="2">−0.07</td>
                <td colspan="2">0.02</td>
                <td>.002</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Age (years, ref=18-25)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td colspan="2">−0.16</td>
                <td colspan="2">0.13</td>
                <td colspan="2">.20</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td colspan="2">−0.10</td>
                <td colspan="2">0.16</td>
                <td colspan="2">.52</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td colspan="2">−0.41</td>
                <td colspan="2">0.26</td>
                <td colspan="2">.12</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Age (years, ref=18-25) × week</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>26-35</td>
                <td colspan="2">−0.03</td>
                <td colspan="2">0.02</td>
                <td colspan="2">.15</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>36-49</td>
                <td colspan="2">−0.01</td>
                <td colspan="2">0.03</td>
                <td colspan="2">.76</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>&#62;50</td>
                <td colspan="2">0.04</td>
                <td colspan="2">0.04</td>
                <td colspan="2">.32</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Intake PHQ-8 score</td>
                <td colspan="2">3.73</td>
                <td colspan="2">0.06</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Intake PHQ-8 score × week</td>
                <td colspan="2">−0.50</td>
                <td colspan="2">0.03</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">Intake PHQ-8 score × week<sup>2</sup></td>
                <td colspan="2">0.03</td>
                <td colspan="2">0.002</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">First time in therapy</td>
                <td colspan="2">−0.07</td>
                <td colspan="2">0.13</td>
                <td>.58</td>
              </tr>
              <tr valign="top">
                <td colspan="3">First time in therapy × week</td>
                <td colspan="2">−0.30</td>
                <td colspan="2">0.06</td>
                <td>&#60;.001</td>
              </tr>
              <tr valign="top">
                <td colspan="3">First time in therapy × week<sup>2</sup></td>
                <td colspan="2">0.03</td>
                <td colspan="2">0.005</td>
                <td>&#60;.001</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table7fn1">
              <p><sup>a</sup>Predictor and predictor × time interaction variables that were significant in simpler models were included. The intake PHQ-8 scores were grand mean-centered.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <fig id="figure4" position="float">
          <label>Figure 4</label>
          <caption>
            <p>Median split Patient Health Questionnaire-8 item (PHQ-8) score at intake and change over time with SE shading.</p>
          </caption>
          <graphic xlink:href="formative_v6i6e36521_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>This is the first population-based study of treatment engagement, subsequent outcomes, and predictors of outcomes in the routine delivery of technology-mediated psychotherapy focused on adult clients experiencing clinically significant levels of depression. Our findings indicate that technology-mediated psychotherapy may be helpful in reducing symptoms of depression. On average, client scores on the PHQ-8 declined over time, dropping below the clinical cut-off from a score of 15 to 10 by week 6. At the same time point, 37% of clients had disengaged from the therapy. There were no differences in the length of treatment engagement or speed of improvement based on whether clients opted for a plan that included synchronous video chat (either once or 4 times per month) in addition to the asynchronous messaging plan.</p>
        <p>Findings from our analyses predicting symptom change and disengagement suggest that the degree to which the therapy was helpful did not depend on the client or therapist characteristics included in the study; however, how quickly clients decided to end the therapy did, though this effect was very small. One exception was education level. Completing formal education beyond high school was related to a sharper decrease in PHQ-8 scores and a shorter length of time before disengagement, which may be interpreted as clients with higher education being a group that is particularly likely to benefit quickly from technology-mediated psychotherapy and, therefore, end it sooner. Consistent with this premise, there is some evidence that individuals with more years of formal education are less likely to end therapy prematurely [<xref ref-type="bibr" rid="ref23">23</xref>]. Clients who were in therapy for the first time were also more likely to improve faster. While this might be associated with less severe depressive symptoms, this result was true even after controlling for baseline depression scores. It may be that those repeatedly seeking treatment have symptoms that are more chronic, even if they are of equal severity to those seeking treatment for the first time.</p>
        <p>The fact that specific age categories and gender were not significantly associated with depressive symptom change is consistent with a study examining moderators of outcomes among primary care patients in the United Kingdom randomized to chat-based CBT therapy versus usual care [<xref ref-type="bibr" rid="ref13">13</xref>]. However, in contrast to our findings, educational attainment did not moderate the relationship between therapy and outcomes in the primary care sample. This is also consistent with a recent meta-regression study examining moderators of outcomes from randomized trials of face-to-face in-person psychotherapy versus a control group among adults with depressive symptoms, in which the authors found no difference between older and younger adults and between men and women [<xref ref-type="bibr" rid="ref24">24</xref>]. The authors note that there were insufficient studies available for the analysis with a low risk of bias to examine education level as a moderator.</p>
        <p>Unfortunately, we did not have data on client race and ethnicity; however, in the meta-regression study mentioned [<xref ref-type="bibr" rid="ref24">24</xref>], studies focusing on diverse samples did not have better or worse effect sizes than the other studies. This is similar to findings from a previous meta-analysis using a similar methodology, looking at randomized controlled trials in which race and ethnicity data were reported [<xref ref-type="bibr" rid="ref25">25</xref>]. The authors examined whether the proportion of adults identified as a racial or ethnic minority in the sample was associated with treatment effect size across 56 trials and found no relation. Although psychotherapy appears to be effective across racial and ethnic groups [<xref ref-type="bibr" rid="ref26">26</xref>], access to and barriers to engagement in psychotherapy are characterized by racial or ethnic disparities [<xref ref-type="bibr" rid="ref27">27</xref>]. For instance, a large meta-analysis of randomized controlled trials of face-to-face in-person psychotherapy for depression among adults found that studies with a higher proportion of participants identifying with a racial or ethnic minority group had higher rates of early therapy termination, as defined by research teams [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
        <p>In a nationally representative US-based study, people identified as Black or African American, Latinx, and Asian Americans were less likely to engage in mental health outpatient care and indicated financial costs to be a major barrier to accessing services [<xref ref-type="bibr" rid="ref29">29</xref>]. Other barriers may be related to beliefs that therapy is not likely to be effective, stigma and shame about needing care and being seen accessing it, language barriers, and cultural practices that may not resonate with a Western medical model [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref30">30</xref>-<xref ref-type="bibr" rid="ref32">32</xref>]. Technology-mediated psychotherapy holds promise for alleviating some of these barriers to psychotherapy, particularly in reducing the need to find a local therapist and attend in-person sessions that may require transportation and additional time to travel to appointments. Although technology-mediated therapy may increase a sense of privacy and confidentiality as clients do not have to present to a clinic or office for care where others may see them, there may be additional barriers to ensuring privacy that have to do with concern over security breaches [<xref ref-type="bibr" rid="ref33">33</xref>]. Although technology and internet access are expanding rapidly, there are still substantial disparities in access to stable and consistent internet and internet-capable smartphones and computers [<xref ref-type="bibr" rid="ref34">34</xref>]. Future research can examine the role of race, ethnicity, and the intersection of access to technology-mediated psychotherapy in both outcomes and therapeutic engagement.</p>
        <p>Our findings also indicate that the length of engagement in services is weakly associated with the variables included in this study. More research should examine the determinants of engagement and work to operationalize them using feasible methods so that clinical care providers can target their efforts to prevent premature disengagement. While the association is weak, our findings indicate that being younger, in therapy for the first time, unmarried but living with their partner (as compared with single), and being with a therapist with less experience predicted a shorter length of time to disengagement from the therapy. Previous research indicates that younger clients are more likely to prematurely end therapy [<xref ref-type="bibr" rid="ref23">23</xref>], and that less experienced therapists tend to have higher premature termination rates [<xref ref-type="bibr" rid="ref23">23</xref>] and more difficulty engaging and retaining clients in therapy [<xref ref-type="bibr" rid="ref35">35</xref>]. Marital status did not appear to be related to psychotherapy or mental health service disengagement in previous face-to-face psychotherapy studies [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref36">36</xref>]. However, a study of chat-based CBT with primary care patients observed that those patients grouped into a category of separated, divorced, or widowed had greater benefit from the therapy than those who were married or living as married and those who were single. The authors determined that the effect seemed to be largely owing to the separated, divorced, or widowed group remaining more symptomatic when waitlisted and obtaining usual care. Understanding the role of marital status in therapeutic gains is complicated by examination of this variable. Owing to the small sample size, categories that seem conceptually similar on the surface may collapse in analyses but may actually represent very distinct groups empirically [<xref ref-type="bibr" rid="ref37">37</xref>]. For instance, being unmarried and living with a partner may hold a unique meaning for the millennial generation, which comprises a large proportion of our sample who are known to delay marriage [<xref ref-type="bibr" rid="ref38">38</xref>], and combining these clients with married clients, as was done in the Button et al [<xref ref-type="bibr" rid="ref13">13</xref>] study, could obscure findings that would emerge if examined separately, as was done in our analysis. Qualitative research may help elucidate the significance of living with a partner, which may be related to a more rapid disengagement from therapy.</p>
        <p>Additional research is needed to understand why client sociodemographic factors may influence perceptions of therapy and decisions to end the therapy. However, it is important to note that not all therapeutic disengagement outcomes are negative. In fact, people are known to vary in how quickly their symptoms respond to psychotherapy, and there is evidence to suggest that people leave therapy when they feel they have a good enough level of change [<xref ref-type="bibr" rid="ref39">39</xref>-<xref ref-type="bibr" rid="ref41">41</xref>]. A previous study of Talkspace clients suggested that this is often the case. Specifically, in a study of clients experiencing depression or anxiety, one-third of those leaving therapy before 12 weeks of enrollment reported reasons for leaving, with 53% indicating satisfaction with reaching therapeutic goals as the reason [<xref ref-type="bibr" rid="ref18">18</xref>]. Therefore, engagement in technology-mediated psychotherapy may require shorter engagement times than in-person session-based therapy, potentially because of the ability of clients to access therapists as needed, rather than the typical once a week or every other week schedule.</p>
        <p>This study examined disengagement from the initial enrollment in therapy; however, it is possible that clients re-engaged, and there may be observable patterns of disengagement and re-engagement over time among the clientele. In addition, the concept of disengagement does not shed light on the nature of a client’s engagement in therapy. Therapeutic engagement can be defined as all efforts made by clients during the course of treatment (both within and between sessions) toward the achievement of changes (treatment outcomes) [<xref ref-type="bibr" rid="ref35">35</xref>]. Once a client has disengaged, it is clear that they no longer participate in the therapy; however, if the client is still in therapy, the degree to which they are genuinely engaged in the process with the therapist can vary substantially and is an important variable in treatment outcomes. In an earlier Talkspace study with depressed or anxious clients, greater amounts of communication between clients in therapists based on word counts across text, audio, and video messaging were related to symptom improvement [<xref ref-type="bibr" rid="ref18">18</xref>]. Research on face-to-face in-person CBT in which the timing of sessions varied suggests that having more frequent sessions in closer proximity to each other versus longer courses of therapy is related to improved outcomes [<xref ref-type="bibr" rid="ref42">42</xref>]. A benefit of technology-mediated psychotherapy, such as what the Talkspace platform offers, is that clients have more potential to increase the intensity of their therapy and perhaps more quickly benefit from it.</p>
        <p>Interestingly, it may not be necessary for clients to meet with their therapist in a more traditional synchronous manner, such as having a face-to-face web-based video-based session, to receive an adequate degree of therapeutic intensity through technology-mediated therapy. In this study, we did not observe a relationship between the type of plan clients opted into and changes in depression scores over time, although some plan types included 1 or 4 synchronous video sessions per month in addition to text and audio or video messaging. In fact, the predominant type of plan clients opted for when they enrolled in the Talkspace platform was asynchronous messaging only (n=5389, 91.5%), although we do not know whether clients opted out of the other plans owing to cost, preference, or both.</p>
      </sec>
      <sec>
        <title>Limitations and Considerations</title>
        <p>A key consideration for our findings is the nature of clients enrolled in technology-mediated psychotherapy. The most common client characteristics were being of the millennial generation [<xref ref-type="bibr" rid="ref38">38</xref>], college educated, single, identifying as female, and having experience with therapy. In addition, as the messaging-only plan was by far the most frequently chosen (91.5%), the study reflects clients willing or preferring to engage in primarily text messaging–based therapy, which may or may not be asynchronous. Men are similarly underrepresented in face-to-face in-person therapy [<xref ref-type="bibr" rid="ref43">43</xref>] and represent a population that may need additional effort to reach and engage in technology-mediated psychotherapy. The population in our study may represent what could be considered early adopters of the innovation of text-based psychotherapy, which is also consistent with the premise that members of the millennial generation are more generally engaged with internet technology and more comfortable using it in myriad ways [<xref ref-type="bibr" rid="ref44">44</xref>]. In addition, data were collected before the COVID-19 pandemic, when engaging in technology-mediated health care was novel for most people. Considering social distancing policies to control the spread of COVID-19, technology-mediated health care proliferated, and the business of web-based therapy grew tremendously as the general demand for psychotherapy simultaneously increased [<xref ref-type="bibr" rid="ref7">7</xref>,<xref ref-type="bibr" rid="ref45">45</xref>]. The postpandemic population may be different from the population included in our study. In fact, a recent study published with postpandemic Talkspace clients suggests that anxiety is a much more common experience and comorbidity among clients since the pandemic started [<xref ref-type="bibr" rid="ref46">46</xref>].</p>
        <p>Our findings for depressive symptom change do not generalize well past the first 6 weeks of therapy, given that missing data on the PHQ-8 increases substantially after 6 weeks. A known contributing factor to missing data on the PHQ-8 is that clients stop taking the measure after disengaging from the therapy. Our estimates of later time points were based on the trajectory that was observed at the time the clients stopped taking the PHQ-8. Given that it is common for people with greater improvement in earlier phases of therapy to end therapy earlier than those whose symptoms do not remit as quickly [<xref ref-type="bibr" rid="ref47">47</xref>], it is possible that the overall improvement in depression symptoms is an overestimation of what we would observe if clients continued taking PHQ-8 assessments. There are high rates of missing data, aside from that, which can be attributed to therapy disengagement. Therefore, these results may only be generalized to people who are receiving therapy and who complete the PHQ-8 assessments. In addition, given that we observed that higher intake PHQ-8 scores were predictive of symptom improvement, a likely caveat for our findings is that there is some regression to the mean, a statistical artifact of more extreme-scoring individuals to score more closely to the average on the next assessment. In randomized controlled trials, where regression to the mean is accounted for, it is more common to actually see higher initial depression scores as related to less symptom improvement [<xref ref-type="bibr" rid="ref48">48</xref>-<xref ref-type="bibr" rid="ref50">50</xref>]. Like all regression models, our curvilinear time modeling provides an overall average estimate of symptom trajectories and is not able to demonstrate what are likely nonlinear relationships with symptoms and time that would be observed if examined at the individual client level [<xref ref-type="bibr" rid="ref51">51</xref>]. However, we can say that, on average, we see a trend for clients to do better symptomatically than when they enrolled.</p>
        <p>We conducted a naturalistic study using routinely collected data from the Talkspace platform. Therefore, there were limitations in the amount and nature of the data available. First, we did not account for other conditions that clients may be seeking therapy for, or conditions that may moderate therapeutic outcomes. For instance, the presence of a personality disorder is related to premature termination of therapy [<xref ref-type="bibr" rid="ref23">23</xref>] and is known to impede symptom improvement [<xref ref-type="bibr" rid="ref52">52</xref>]. The PHQ-8 does not capture suicidal thinking, so we do not have estimates of suicide risk using the item from the PHQ-9 that asks about <italic>thoughts that you would be better off dead, or of hurting yourself</italic>, which is known to be predictive of suicidal behavior and a common question asked as part of routine suicide risk screening procedures [<xref ref-type="bibr" rid="ref53">53</xref>]. In addition, we do not know what type of therapy is provided by licensed clinicians; however, therapy from various theoretical orientations is known to be effective for depression [<xref ref-type="bibr" rid="ref54">54</xref>]. The availability of textual data allows for an ample exploration of what is happening in routine psychotherapy in this context. Future research can use the rich data made available through text messages and transcripts made from audio or video interactions to examine the degree to which evidence-based interventions and practices are part of the therapy and how this might be related to disengagement, engagement, and symptom outcomes.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>Technology-mediated psychotherapy is becoming an increasingly popular alternative to in-person psychotherapy. Our findings add to the growing literature showing the benefits of technology-mediated psychotherapy in general and messaging-based therapy in particular. In addition, benefits were observed over a relatively brief period, with a reduction in symptoms observed within 16 weeks of therapy. Some populations may require additional efforts to remain engaged once they enroll, such as in younger adults. Future research is needed to examine how well technology-mediated psychotherapy addresses disparities in access to therapy and opportunities to enhance reach and ensure the effectiveness of this treatment modality across the full spectrum of adults experiencing depression.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">CBT</term>
          <def>
            <p>cognitive behavior therapy</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">OR</term>
          <def>
            <p>odds ratio</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">PHQ</term>
          <def>
            <p>Patient Health Questionnaire</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This research was supported by the National Institute of Mental Health of the National Institutes of Health under award numbers K23MH118361, P50MH115837, and R44MH124334. The content is the sole responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.</p>
    </ack>
    <fn-group>
      <fn fn-type="con">
        <p>DD, MDP, TDH, and PA contributed to the conceptualization, review, and writing of the manuscript. DD, MDP, and SC formulated the data analytic approach. Data analysis was conducted using MDP and SC. DD, MDP, TDH, and PA drafted portions of the manuscript and provided substantive review of manuscript drafts. DD coordinated the manuscript preparation and revisions.</p>
      </fn>
      <fn fn-type="conflict">
        <p>TDH is an employee of the service that provided the data.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="book">
          <source>Key Substance Use and Mental Health Indicators in the United States: Results from the 2019 National Survey on Drug Use and Health</source>
          <year>2020</year>
          <publisher-loc>Rockville, MD</publisher-loc>
          <publisher-name>Center for Behavioral Health Statistics and Quality, Substance Abuse and Mental Health Services Administration</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kessler</surname>
              <given-names>RC</given-names>
            </name>
            <name name-style="western">
              <surname>Petukhova</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Sampson</surname>
              <given-names>NA</given-names>
            </name>
            <name name-style="western">
              <surname>Zaslavsky</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Wittchen</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Twelve-month and lifetime prevalence and lifetime morbid risk of anxiety and mood disorders in the United States</article-title>
          <source>Int J Methods Psychiatr Res</source>
          <year>2012</year>
          <month>09</month>
          <volume>21</volume>
          <issue>3</issue>
          <fpage>169</fpage>
          <lpage>84</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22865617"/>
          </comment>
          <pub-id pub-id-type="doi">10.1002/mpr.1359</pub-id>
          <pub-id pub-id-type="medline">22865617</pub-id>
          <pub-id pub-id-type="pmcid">PMC4005415</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Psychotherapies for adult depression: recent developments</article-title>
          <source>Curr Opin Psychiatry</source>
          <year>2015</year>
          <month>01</month>
          <volume>28</volume>
          <issue>1</issue>
          <fpage>24</fpage>
          <lpage>9</lpage>
          <pub-id pub-id-type="doi">10.1097/YCO.0000000000000121</pub-id>
          <pub-id pub-id-type="medline">25415495</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McHugh</surname>
              <given-names>RK</given-names>
            </name>
            <name name-style="western">
              <surname>Whitton</surname>
              <given-names>SW</given-names>
            </name>
            <name name-style="western">
              <surname>Peckham</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Welge</surname>
              <given-names>JA</given-names>
            </name>
            <name name-style="western">
              <surname>Otto</surname>
              <given-names>MW</given-names>
            </name>
          </person-group>
          <article-title>Patient preference for psychological vs pharmacologic treatment of psychiatric disorders</article-title>
          <source>J Clin Psychiatry</source>
          <year>2013</year>
          <month>06</month>
          <day>15</day>
          <volume>74</volume>
          <issue>06</issue>
          <fpage>595</fpage>
          <lpage>602</lpage>
          <pub-id pub-id-type="doi">10.4088/jcp.12r07757</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mohr</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Hart</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Howard</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Julian</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Vella</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Catledge</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Feldman</surname>
              <given-names>MD</given-names>
            </name>
          </person-group>
          <article-title>Barriers to psychotherapy among depressed and nondepressed primary care patients</article-title>
          <source>Ann Behav Med</source>
          <year>2006</year>
          <month>12</month>
          <volume>32</volume>
          <issue>3</issue>
          <fpage>254</fpage>
          <lpage>8</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pubmed.ncbi.nlm.nih.gov/17107299/"/>
          </comment>
          <pub-id pub-id-type="doi">10.1207/s15324796abm3203_12</pub-id>
          <pub-id pub-id-type="medline">17107299</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Imel</surname>
              <given-names>ZE</given-names>
            </name>
            <name name-style="western">
              <surname>Caperton</surname>
              <given-names>DD</given-names>
            </name>
            <name name-style="western">
              <surname>Tanana</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Atkins</surname>
              <given-names>DC</given-names>
            </name>
          </person-group>
          <article-title>Technology-enhanced human interaction in psychotherapy</article-title>
          <source>J Couns Psychol</source>
          <year>2017</year>
          <month>07</month>
          <volume>64</volume>
          <issue>4</issue>
          <fpage>385</fpage>
          <lpage>93</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/28318277"/>
          </comment>
          <pub-id pub-id-type="doi">10.1037/cou0000213</pub-id>
          <pub-id pub-id-type="medline">28318277</pub-id>
          <pub-id pub-id-type="pii">2017-12467-001</pub-id>
          <pub-id pub-id-type="pmcid">PMC5491349</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Steele</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Pandemic, new platforms prompt surge in new therapists</article-title>
          <source>Wall Street Journal</source>
          <year>2020</year>
          <access-date>2021-12-28</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.wsj.com/articles/pandemic-new-platforms-prompt-surge-in-new-therapists-11601816400">https://www.wsj.com/articles/pandemic-new-platforms-prompt-surge-in-new-therapists-11601816400</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Castro</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gili</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ricci-Cabello</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Roca</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Gilbody</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Perez-Ara</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Seguí</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>McMillan</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Effectiveness and adherence of telephone-administered psychotherapy for depression: a systematic review and meta-analysis</article-title>
          <source>J Affect Disord</source>
          <year>2020</year>
          <month>01</month>
          <day>01</day>
          <volume>260</volume>
          <fpage>514</fpage>
          <lpage>26</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jad.2019.09.023</pub-id>
          <pub-id pub-id-type="medline">31539688</pub-id>
          <pub-id pub-id-type="pii">S0165-0327(19)31172-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Osenbach</surname>
              <given-names>JE</given-names>
            </name>
            <name name-style="western">
              <surname>O'Brien</surname>
              <given-names>KM</given-names>
            </name>
            <name name-style="western">
              <surname>Mishkind</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Smolenski</surname>
              <given-names>DJ</given-names>
            </name>
          </person-group>
          <article-title>Synchronous telehealth technologies in psychotherapy for depression: a meta-analysis</article-title>
          <source>Depress Anxiety</source>
          <year>2013</year>
          <month>11</month>
          <day>06</day>
          <volume>30</volume>
          <issue>11</issue>
          <fpage>1058</fpage>
          <lpage>67</lpage>
          <pub-id pub-id-type="doi">10.1002/da.22165</pub-id>
          <pub-id pub-id-type="medline">23922191</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mohr</surname>
              <given-names>DC</given-names>
            </name>
            <name name-style="western">
              <surname>Vella</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Hart</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Heckman</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Simon</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>The effect of telephone-administered psychotherapy on symptoms of depression and attrition: a meta-analysis</article-title>
          <source>Clin Psychol (New York)</source>
          <year>2008</year>
          <volume>15</volume>
          <issue>3</issue>
          <fpage>243</fpage>
          <lpage>53</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/21369344"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/j.1468-2850.2008.00134.x</pub-id>
          <pub-id pub-id-type="medline">21369344</pub-id>
          <pub-id pub-id-type="pmcid">PMC3045729</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vernmark</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Lenndin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bjärehed</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Carlsson</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Karlsson</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Oberg</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Carlbring</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Eriksson</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Internet administered guided self-help versus individualized e-mail therapy: a randomized trial of two versions of CBT for major depression</article-title>
          <source>Behav Res Ther</source>
          <year>2010</year>
          <month>05</month>
          <volume>48</volume>
          <issue>5</issue>
          <fpage>368</fpage>
          <lpage>76</lpage>
          <pub-id pub-id-type="doi">10.1016/j.brat.2010.01.005</pub-id>
          <pub-id pub-id-type="medline">20152960</pub-id>
          <pub-id pub-id-type="pii">S0005-7967(10)00006-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kessler</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Lewis</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Kaur</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wiles</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>King</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Weich</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sharp</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Araya</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Hollinghurst</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Peters</surname>
              <given-names>TJ</given-names>
            </name>
          </person-group>
          <article-title>Therapist-delivered internet psychotherapy for depression in primary care: a randomised controlled trial</article-title>
          <source>Lancet</source>
          <year>2009</year>
          <month>08</month>
          <volume>374</volume>
          <issue>9690</issue>
          <fpage>628</fpage>
          <lpage>34</lpage>
          <pub-id pub-id-type="doi">10.1016/s0140-6736(09)61257-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Button</surname>
              <given-names>KS</given-names>
            </name>
            <name name-style="western">
              <surname>Wiles</surname>
              <given-names>NJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lewis</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Peters</surname>
              <given-names>TJ</given-names>
            </name>
            <name name-style="western">
              <surname>Kessler</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Factors associated with differential response to online cognitive behavioural therapy</article-title>
          <source>Soc Psychiatry Psychiatr Epidemiol</source>
          <year>2012</year>
          <month>05</month>
          <day>4</day>
          <volume>47</volume>
          <issue>5</issue>
          <fpage>827</fpage>
          <lpage>33</lpage>
          <pub-id pub-id-type="doi">10.1007/s00127-011-0389-1</pub-id>
          <pub-id pub-id-type="medline">21541696</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barak</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hen</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Boniel-Nissim</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Shapira</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>A comprehensive review and a meta-analysis of the effectiveness of internet-based psychotherapeutic interventions</article-title>
          <source>J Technol Human Service</source>
          <year>2008</year>
          <month>07</month>
          <day>03</day>
          <volume>26</volume>
          <issue>2-4</issue>
          <fpage>109</fpage>
          <lpage>60</lpage>
          <pub-id pub-id-type="doi">10.1080/15228830802094429</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nitzburg</surname>
              <given-names>GC</given-names>
            </name>
            <name name-style="western">
              <surname>Farber</surname>
              <given-names>BA</given-names>
            </name>
          </person-group>
          <article-title>Patterns of utilization and a case illustration of an interactive text-based psychotherapy delivery system</article-title>
          <source>J Clin Psychol</source>
          <year>2019</year>
          <month>02</month>
          <day>10</day>
          <volume>75</volume>
          <issue>2</issue>
          <fpage>247</fpage>
          <lpage>59</lpage>
          <pub-id pub-id-type="doi">10.1002/jclp.22718</pub-id>
          <pub-id pub-id-type="medline">30628062</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hull</surname>
              <given-names>TD</given-names>
            </name>
            <name name-style="western">
              <surname>Mahan</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>A study of asynchronous mobile-enabled SMS text psychotherapy</article-title>
          <source>Telemed J E Health</source>
          <year>2017</year>
          <month>03</month>
          <volume>23</volume>
          <issue>3</issue>
          <fpage>240</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1089/tmj.2016.0114</pub-id>
          <pub-id pub-id-type="medline">27797646</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>DellaCrosse</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Mahan</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Hull</surname>
              <given-names>TD</given-names>
            </name>
          </person-group>
          <article-title>The effect of messaging therapy for depression and anxiety on employee productivity</article-title>
          <source>J technol behav sci</source>
          <year>2018</year>
          <month>6</month>
          <day>26</day>
          <volume>4</volume>
          <issue>1</issue>
          <fpage>1</fpage>
          <lpage>5</lpage>
          <pub-id pub-id-type="doi">10.1007/s41347-018-0064-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hull</surname>
              <given-names>TD</given-names>
            </name>
            <name name-style="western">
              <surname>Malgaroli</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Connolly</surname>
              <given-names>PS</given-names>
            </name>
            <name name-style="western">
              <surname>Feuerstein</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Simon</surname>
              <given-names>NM</given-names>
            </name>
          </person-group>
          <article-title>Two-way messaging therapy for depression and anxiety: longitudinal response trajectories</article-title>
          <source>BMC Psychiatry</source>
          <year>2020</year>
          <month>06</month>
          <day>12</day>
          <volume>20</volume>
          <issue>1</issue>
          <fpage>297</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcpsychiatry.biomedcentral.com/articles/10.1186/s12888-020-02721-x"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12888-020-02721-x</pub-id>
          <pub-id pub-id-type="medline">32532225</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12888-020-02721-x</pub-id>
          <pub-id pub-id-type="pmcid">PMC7291694</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Posner</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>GK</given-names>
            </name>
            <name name-style="western">
              <surname>Stanley</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Brent</surname>
              <given-names>DA</given-names>
            </name>
            <name name-style="western">
              <surname>Yershova</surname>
              <given-names>KV</given-names>
            </name>
            <name name-style="western">
              <surname>Oquendo</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Currier</surname>
              <given-names>GW</given-names>
            </name>
            <name name-style="western">
              <surname>Melvin</surname>
              <given-names>GA</given-names>
            </name>
            <name name-style="western">
              <surname>Greenhill</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Shen</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Mann</surname>
              <given-names>JJ</given-names>
            </name>
          </person-group>
          <article-title>The Columbia-Suicide Severity Rating Scale: initial validity and internal consistency findings from three multisite studies with adolescents and adults</article-title>
          <source>Am J Psychiatry</source>
          <year>2011</year>
          <month>12</month>
          <volume>168</volume>
          <issue>12</issue>
          <fpage>1266</fpage>
          <lpage>77</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22193671"/>
          </comment>
          <pub-id pub-id-type="doi">10.1176/appi.ajp.2011.10111704</pub-id>
          <pub-id pub-id-type="medline">22193671</pub-id>
          <pub-id pub-id-type="pmcid">PMC3893686</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kroenke</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Strine</surname>
              <given-names>TW</given-names>
            </name>
            <name name-style="western">
              <surname>Spitzer</surname>
              <given-names>RL</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>JB</given-names>
            </name>
            <name name-style="western">
              <surname>Berry</surname>
              <given-names>JT</given-names>
            </name>
            <name name-style="western">
              <surname>Mokdad</surname>
              <given-names>AH</given-names>
            </name>
          </person-group>
          <article-title>The PHQ-8 as a measure of current depression in the general population</article-title>
          <source>J Affect Disord</source>
          <year>2009</year>
          <month>04</month>
          <volume>114</volume>
          <issue>1-3</issue>
          <fpage>163</fpage>
          <lpage>73</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jad.2008.06.026</pub-id>
          <pub-id pub-id-type="medline">18752852</pub-id>
          <pub-id pub-id-type="pii">S0165-0327(08)00282-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Löwe</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Unützer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Callahan</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Perkins</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Kroenke</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Monitoring depression treatment outcomes with the patient health questionnaire-9</article-title>
          <source>Med Care</source>
          <year>2004</year>
          <month>12</month>
          <volume>42</volume>
          <issue>12</issue>
          <fpage>1194</fpage>
          <lpage>201</lpage>
          <pub-id pub-id-type="doi">10.1097/00005650-200412000-00006</pub-id>
          <pub-id pub-id-type="medline">15550799</pub-id>
          <pub-id pub-id-type="pii">00005650-200412000-00006</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Singer</surname>
              <given-names>JD</given-names>
            </name>
            <name name-style="western">
              <surname>Willett</surname>
              <given-names>JB</given-names>
            </name>
          </person-group>
          <source>Applied Longitudinal Data Analysis: Modeling Change and Event Occurrence</source>
          <year>2003</year>
          <publisher-loc>Oxford, UK</publisher-loc>
          <publisher-name>Oxford Scholarship</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Swift</surname>
              <given-names>JK</given-names>
            </name>
            <name name-style="western">
              <surname>Greenberg</surname>
              <given-names>RP</given-names>
            </name>
          </person-group>
          <article-title>Premature discontinuation in adult psychotherapy: a meta-analysis</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2012</year>
          <month>08</month>
          <volume>80</volume>
          <issue>4</issue>
          <fpage>547</fpage>
          <lpage>59</lpage>
          <pub-id pub-id-type="doi">10.1037/a0028226</pub-id>
          <pub-id pub-id-type="medline">22506792</pub-id>
          <pub-id pub-id-type="pii">2012-10020-001</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Karyotaki</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Reijnders</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Huibers</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>Who benefits from psychotherapies for adult depression? A meta-analytic update of the evidence</article-title>
          <source>Cogn Behav Ther</source>
          <year>2018</year>
          <month>03</month>
          <day>18</day>
          <volume>47</volume>
          <issue>2</issue>
          <fpage>91</fpage>
          <lpage>106</lpage>
          <pub-id pub-id-type="doi">10.1080/16506073.2017.1420098</pub-id>
          <pub-id pub-id-type="medline">29345530</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ünlü Ince</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Riper</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>van 't Hof</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>The effects of psychotherapy on depression among racial-ethnic minority groups: a metaregression analysis</article-title>
          <source>Psychiatr Serv</source>
          <year>2014</year>
          <month>05</month>
          <day>01</day>
          <volume>65</volume>
          <issue>5</issue>
          <fpage>612</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1176/appi.ps.201300165</pub-id>
          <pub-id pub-id-type="medline">24535615</pub-id>
          <pub-id pub-id-type="pii">1831969</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Huey</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Tilley</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Jones</surname>
              <given-names>EO</given-names>
            </name>
            <name name-style="western">
              <surname>Smith</surname>
              <given-names>CA</given-names>
            </name>
          </person-group>
          <article-title>The contribution of cultural competence to evidence-based care for ethnically diverse populations</article-title>
          <source>Annu Rev Clin Psychol</source>
          <year>2014</year>
          <month>03</month>
          <day>28</day>
          <volume>10</volume>
          <issue>1</issue>
          <fpage>305</fpage>
          <lpage>38</lpage>
          <pub-id pub-id-type="doi">10.1146/annurev-clinpsy-032813-153729</pub-id>
          <pub-id pub-id-type="medline">24437436</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tambling</surname>
              <given-names>RR</given-names>
            </name>
            <name name-style="western">
              <surname>Hynes</surname>
              <given-names>KC</given-names>
            </name>
            <name name-style="western">
              <surname>D’Aniello</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Are barriers to psychotherapy treatment seeking indicators of social determinants of health?: a critical review of the literature</article-title>
          <source>Am J Family Ther</source>
          <year>2021</year>
          <month>01</month>
          <day>08</day>
          <fpage>1</fpage>
          <lpage>16</lpage>
          <pub-id pub-id-type="doi">10.1080/01926187.2020.1867931</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cooper</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Conklin</surname>
              <given-names>LR</given-names>
            </name>
          </person-group>
          <article-title>Dropout from individual psychotherapy for major depression: a meta-analysis of randomized clinical trials</article-title>
          <source>Clin Psychol Rev</source>
          <year>2015</year>
          <month>08</month>
          <volume>40</volume>
          <fpage>57</fpage>
          <lpage>65</lpage>
          <pub-id pub-id-type="doi">10.1016/j.cpr.2015.05.001</pub-id>
          <pub-id pub-id-type="medline">26067572</pub-id>
          <pub-id pub-id-type="pii">S0272-7358(15)00061-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="web">
          <article-title>Racial/ethnic differences in mental health service use among adults</article-title>
          <source>Substance Abuse and Mental Health Services Administration</source>
          <access-date>2021-12-27</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.samhsa.gov/data/report/racialethnic-differences-mental-health-service-use-among-adults">https://www.samhsa.gov/data/report/racialethnic-differences-mental-health-service-use-among-adults</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barrera</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Longoria</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Examining cultural mental health care barriers among Latinos</article-title>
          <source>J Leadership Equity Res</source>
          <year>2018</year>
          <volume>4</volume>
          <issue>1</issue>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rizzo</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Racial and ethnic disparities in use of psychotherapy: evidence from U.S. national survey data</article-title>
          <source>Psychiatr Serv</source>
          <year>2010</year>
          <month>04</month>
          <volume>61</volume>
          <issue>4</issue>
          <fpage>364</fpage>
          <lpage>72</lpage>
          <pub-id pub-id-type="doi">10.1176/ps.2010.61.4.364</pub-id>
          <pub-id pub-id-type="medline">20360275</pub-id>
          <pub-id pub-id-type="pii">61/4/364</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>VL</given-names>
            </name>
            <name name-style="western">
              <surname>Bazile</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Akbar</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>African Americans' perceptions of psychotherapy and psychotherapists</article-title>
          <source>Professional Psychol Res Practice</source>
          <year>2004</year>
          <volume>35</volume>
          <issue>1</issue>
          <fpage>19</fpage>
          <lpage>26</lpage>
          <pub-id pub-id-type="doi">10.1037/0735-7028.35.1.19</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Renn</surname>
              <given-names>BN</given-names>
            </name>
            <name name-style="western">
              <surname>Hoeft</surname>
              <given-names>TJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>HS</given-names>
            </name>
            <name name-style="western">
              <surname>Bauer</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Areán</surname>
              <given-names>PA</given-names>
            </name>
          </person-group>
          <article-title>Preference for in-person psychotherapy versus digital psychotherapy options for depression: survey of adults in the U.S</article-title>
          <source>NPJ Digit Med</source>
          <year>2019</year>
          <month>2</month>
          <day>11</day>
          <volume>2</volume>
          <issue>1</issue>
          <fpage>6</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41746-019-0077-1"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41746-019-0077-1</pub-id>
          <pub-id pub-id-type="medline">31304356</pub-id>
          <pub-id pub-id-type="pii">77</pub-id>
          <pub-id pub-id-type="pmcid">PMC6550152</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Marler</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Mobile phones and inequality: findings, trends, and future directions</article-title>
          <source>New Media Soc</source>
          <year>2018</year>
          <month>04</month>
          <day>07</day>
          <volume>20</volume>
          <issue>9</issue>
          <fpage>3498</fpage>
          <lpage>520</lpage>
          <pub-id pub-id-type="doi">10.1177/1461444818765154</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Holdsworth</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Bowen</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Howat</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Client engagement in psychotherapeutic treatment and associations with client characteristics, therapist characteristics, and treatment factors</article-title>
          <source>Clin Psychol Rev</source>
          <year>2014</year>
          <month>07</month>
          <volume>34</volume>
          <issue>5</issue>
          <fpage>428</fpage>
          <lpage>50</lpage>
          <pub-id pub-id-type="doi">10.1016/j.cpr.2014.06.004</pub-id>
          <pub-id pub-id-type="medline">25000204</pub-id>
          <pub-id pub-id-type="pii">S0272-7358(14)00096-8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>O'Brien</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Fahmy</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>SP</given-names>
            </name>
          </person-group>
          <article-title>Disengagement from mental health services. A literature review</article-title>
          <source>Soc Psychiatry Psychiatr Epidemiol</source>
          <year>2009</year>
          <month>07</month>
          <day>26</day>
          <volume>44</volume>
          <issue>7</issue>
          <fpage>558</fpage>
          <lpage>68</lpage>
          <pub-id pub-id-type="doi">10.1007/s00127-008-0476-0</pub-id>
          <pub-id pub-id-type="medline">19037573</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Afifi</surname>
              <given-names>TO</given-names>
            </name>
            <name name-style="western">
              <surname>Cox</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Enns</surname>
              <given-names>MW</given-names>
            </name>
          </person-group>
          <article-title>Mental health profiles among married, never-married, and separated/divorced mothers in a nationally representative sample</article-title>
          <source>Soc Psychiatry Psychiatr Epidemiol</source>
          <year>2006</year>
          <month>02</month>
          <day>9</day>
          <volume>41</volume>
          <issue>2</issue>
          <fpage>122</fpage>
          <lpage>9</lpage>
          <pub-id pub-id-type="doi">10.1007/s00127-005-0005-3</pub-id>
          <pub-id pub-id-type="medline">16467954</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="web">
          <article-title>Millennial life: how young adulthood today compares with prior generations</article-title>
          <source>Pew Research Center</source>
          <access-date>2021-12-27</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.researchgate.net/publication/337113826_Millennial_life_How_young_adulthood_today_compares_with_prior_generations">https://www.researchgate.net/publication/337113826_Millennial_life_How_young_adulthood_today_compares_with_prior_generations</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Sripada</surname>
              <given-names>RK</given-names>
            </name>
            <name name-style="western">
              <surname>Hale</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Ganoczy</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Trivedi</surname>
              <given-names>RB</given-names>
            </name>
            <name name-style="western">
              <surname>Arnow</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Pfeiffer</surname>
              <given-names>PN</given-names>
            </name>
          </person-group>
          <article-title>Psychotherapy and depressive symptom trajectories among VA patients: comparing dose-effect and good-enough level models</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2021</year>
          <month>05</month>
          <volume>89</volume>
          <issue>5</issue>
          <fpage>379</fpage>
          <lpage>92</lpage>
          <pub-id pub-id-type="doi">10.1037/ccp0000645</pub-id>
          <pub-id pub-id-type="medline">34124925</pub-id>
          <pub-id pub-id-type="pii">2021-54176-001</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Simon</surname>
              <given-names>GE</given-names>
            </name>
            <name name-style="western">
              <surname>Imel</surname>
              <given-names>ZE</given-names>
            </name>
            <name name-style="western">
              <surname>Ludman</surname>
              <given-names>EJ</given-names>
            </name>
            <name name-style="western">
              <surname>Steinfeld</surname>
              <given-names>BJ</given-names>
            </name>
          </person-group>
          <article-title>Is dropout after a first psychotherapy visit always a bad outcome?</article-title>
          <source>Psychiatr Serv</source>
          <year>2012</year>
          <month>07</month>
          <volume>63</volume>
          <issue>7</issue>
          <fpage>705</fpage>
          <lpage>7</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22752034"/>
          </comment>
          <pub-id pub-id-type="doi">10.1176/appi.ps.201100309</pub-id>
          <pub-id pub-id-type="medline">22752034</pub-id>
          <pub-id pub-id-type="pii">1204710</pub-id>
          <pub-id pub-id-type="pmcid">PMC3708593</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Barkham</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Connell</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Stiles</surname>
              <given-names>WB</given-names>
            </name>
            <name name-style="western">
              <surname>Miles</surname>
              <given-names>JN</given-names>
            </name>
            <name name-style="western">
              <surname>Margison</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Evans</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Mellor-Clark</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Dose-effect relations and responsive regulation of treatment duration: the good enough level</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2006</year>
          <volume>74</volume>
          <issue>1</issue>
          <fpage>160</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1037/0022-006x.74.1.160</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bruijniks</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lemmens</surname>
              <given-names>LH</given-names>
            </name>
            <name name-style="western">
              <surname>Hollon</surname>
              <given-names>SD</given-names>
            </name>
            <name name-style="western">
              <surname>Peeters</surname>
              <given-names>FP</given-names>
            </name>
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Arntz</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Dingemanse</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Willems</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>van Oppen</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Twisk</surname>
              <given-names>JW</given-names>
            </name>
            <name name-style="western">
              <surname>van den Boogaard</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Spijker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bosmans</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Huibers</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>The effects of once- versus twice-weekly sessions on psychotherapy outcomes in depressed patients</article-title>
          <source>Br J Psychiatry</source>
          <year>2020</year>
          <month>04</month>
          <day>07</day>
          <volume>216</volume>
          <issue>4</issue>
          <fpage>222</fpage>
          <lpage>30</lpage>
          <pub-id pub-id-type="doi">10.1192/bjp.2019.265</pub-id>
          <pub-id pub-id-type="medline">32029012</pub-id>
          <pub-id pub-id-type="pii">S0007125019002654</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Seidler</surname>
              <given-names>ZE</given-names>
            </name>
            <name name-style="western">
              <surname>Rice</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Ogrodniczuk</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Oliffe</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Dhillon</surname>
              <given-names>HM</given-names>
            </name>
          </person-group>
          <article-title>Men, masculinities, depression: implications for mental health services from a Delphi expert consensus study</article-title>
          <source>Professional Psychol Res Pract</source>
          <year>2019</year>
          <month>02</month>
          <volume>50</volume>
          <issue>1</issue>
          <fpage>51</fpage>
          <lpage>61</lpage>
          <pub-id pub-id-type="doi">10.1037/pro0000220</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vogels</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Millennials stand out for their technology use, but older generations also embrace digital life</article-title>
          <source>Pew Research Center</source>
          <access-date>2021-12-27</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.pewresearch.org/fact-tank/2019/09/09/us-generations-technology-use/">https://www.pewresearch.org/fact-tank/2019/09/09/us-generations-technology-use/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="web">
          <article-title>Worsening mental health crisis pressures psychologist workforce COVID-19 Practitioner Survey</article-title>
          <source>American Psychological Association</source>
          <access-date>2021-12-27</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.apa.org/pubs/reports/practitioner/covid-19-2021">https://www.apa.org/pubs/reports/practitioner/covid-19-2021</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hull</surname>
              <given-names>TD</given-names>
            </name>
            <name name-style="western">
              <surname>Levine</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Bantilan</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Desai</surname>
              <given-names>AN</given-names>
            </name>
            <name name-style="western">
              <surname>Majumder</surname>
              <given-names>MS</given-names>
            </name>
          </person-group>
          <article-title>Analyzing digital evidence from a telemental health platform to assess complex psychological responses to the COVID-19 pandemic: content analysis of text messages</article-title>
          <source>JMIR Form Res</source>
          <year>2021</year>
          <month>02</month>
          <day>09</day>
          <volume>5</volume>
          <issue>2</issue>
          <fpage>e26190</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://formative.jmir.org/2021/2/e26190/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/26190</pub-id>
          <pub-id pub-id-type="medline">33502999</pub-id>
          <pub-id pub-id-type="pii">v5i2e26190</pub-id>
          <pub-id pub-id-type="pmcid">PMC7879721</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Niileksela</surname>
              <given-names>CR</given-names>
            </name>
            <name name-style="western">
              <surname>Ghosh</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Janis</surname>
              <given-names>RA</given-names>
            </name>
          </person-group>
          <article-title>The dose-effect and good enough level models of change for specific psychological concerns</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2021</year>
          <month>03</month>
          <volume>89</volume>
          <issue>3</issue>
          <fpage>200</fpage>
          <lpage>13</lpage>
          <pub-id pub-id-type="doi">10.1037/ccp0000635</pub-id>
          <pub-id pub-id-type="medline">33829808</pub-id>
          <pub-id pub-id-type="pii">2021-33935-006</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Elkin</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Gibbons</surname>
              <given-names>RD</given-names>
            </name>
            <name name-style="western">
              <surname>Shea</surname>
              <given-names>MT</given-names>
            </name>
            <name name-style="western">
              <surname>Sotsky</surname>
              <given-names>SM</given-names>
            </name>
            <name name-style="western">
              <surname>Watkins</surname>
              <given-names>JT</given-names>
            </name>
            <name name-style="western">
              <surname>Pilkonis</surname>
              <given-names>PA</given-names>
            </name>
            <name name-style="western">
              <surname>Hedeker</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Initial severity and differential treatment outcome in the National Institute of Mental Health Treatment of Depression Collaborative Research Program</article-title>
          <source>J Consult Clin Psychol</source>
          <year>1995</year>
          <volume>63</volume>
          <issue>5</issue>
          <fpage>841</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1037/0022-006x.63.5.841</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Delgadillo</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Moreea</surname>
              <given-names>O</given-names>
            </name>
            <name name-style="western">
              <surname>Lutz</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>Different people respond differently to therapy: a demonstration using patient profiling and risk stratification</article-title>
          <source>Behav Res Ther</source>
          <year>2016</year>
          <month>04</month>
          <volume>79</volume>
          <fpage>15</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.1016/j.brat.2016.02.003</pub-id>
          <pub-id pub-id-type="medline">26937855</pub-id>
          <pub-id pub-id-type="pii">S0005-7967(16)30024-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Katon</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Unützer</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Russo</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Major depression: the importance of clinical characteristics and treatment response to prognosis</article-title>
          <source>Depress Anxiety</source>
          <year>2010</year>
          <month>01</month>
          <volume>27</volume>
          <issue>1</issue>
          <fpage>19</fpage>
          <lpage>26</lpage>
          <pub-id pub-id-type="doi">10.1002/da.20613</pub-id>
          <pub-id pub-id-type="medline">19798766</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Helmich</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Wichers</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Olthof</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Strunk</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Aas</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Aichhorn</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Schiepek</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Snippe</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Sudden gains in day-to-day change: revealing nonlinear patterns of individual improvement in depression</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2020</year>
          <month>02</month>
          <volume>88</volume>
          <issue>2</issue>
          <fpage>119</fpage>
          <lpage>27</lpage>
          <pub-id pub-id-type="doi">10.1037/ccp0000469</pub-id>
          <pub-id pub-id-type="medline">31894994</pub-id>
          <pub-id pub-id-type="pii">2019-80727-002</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Newton-Howes</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Tyrer</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Johnson</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Personality disorder and the outcome of depression: meta-analysis of published studies</article-title>
          <source>Br J Psychiatry</source>
          <year>2006</year>
          <month>01</month>
          <day>02</day>
          <volume>188</volume>
          <issue>1</issue>
          <fpage>13</fpage>
          <lpage>20</lpage>
          <pub-id pub-id-type="doi">10.1192/bjp.188.1.13</pub-id>
          <pub-id pub-id-type="medline">16388064</pub-id>
          <pub-id pub-id-type="pii">S0007125000169107</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref53">
        <label>53</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Simon</surname>
              <given-names>GE</given-names>
            </name>
            <name name-style="western">
              <surname>Rutter</surname>
              <given-names>CM</given-names>
            </name>
            <name name-style="western">
              <surname>Peterson</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Oliver</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Whiteside</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Operskalski</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Ludman</surname>
              <given-names>EJ</given-names>
            </name>
          </person-group>
          <article-title>Does response on the PHQ-9 Depression Questionnaire predict subsequent suicide attempt or suicide death?</article-title>
          <source>Psychiatr Serv</source>
          <year>2013</year>
          <month>12</month>
          <day>01</day>
          <volume>64</volume>
          <issue>12</issue>
          <fpage>1195</fpage>
          <lpage>202</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24036589"/>
          </comment>
          <pub-id pub-id-type="doi">10.1176/appi.ps.201200587</pub-id>
          <pub-id pub-id-type="medline">24036589</pub-id>
          <pub-id pub-id-type="pii">1738338</pub-id>
          <pub-id pub-id-type="pmcid">PMC4086215</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref54">
        <label>54</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cuijpers</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>van Straten</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>van Oppen</surname>
              <given-names>P</given-names>
            </name>
          </person-group>
          <article-title>Psychotherapy for depression in adults: a meta-analysis of comparative outcome studies</article-title>
          <source>J Consult Clin Psychol</source>
          <year>2008</year>
          <month>12</month>
          <volume>76</volume>
          <issue>6</issue>
          <fpage>909</fpage>
          <lpage>22</lpage>
          <pub-id pub-id-type="doi">10.1037/a0013075</pub-id>
          <pub-id pub-id-type="medline">19045960</pub-id>
          <pub-id pub-id-type="pii">2008-16943-011</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
