501 data collection changes

bin/test-run

@@ -167,6 +167,7 @@ sub do_send_example_message {
 
     $options->{base_url} = $base_url;
     my $repinfo = {repname => $CAMBRIDGE_MP};
+    $options->{do_post_confirm_survey} = 1;
 
     send_message_to_rep(0, 
         'CB23QJ', # The Guildhall, Cambridge 

db/0001-new-data-collection-columns.sql

@@ -0,0 +1,13 @@
+begin;
+
+  create table analysis_data (
+      message_id char(20) not null references message(id) on delete cascade,
+      message_summary text,
+      analysis_data jsonb,
+      whenanswered timestamp with time zone
+  );
+
+  create unique index analysis_data_message_id_idx
+      on analysis_data(message_id);
+
+commit;

db/schema.sql

@@ -178,6 +178,18 @@ create index message_extradata_message_id_idx
 create unique index message_extradata_message_id_name_idx
     on message_extradata(message_id, name);
 
+-- analysis_data
+-- store post confirmation survey data for analysis
+create table analysis_data (
+    message_id char(20) not null references message(id) on delete cascade,
+    message_summary text,
+    analysis_data jsonb,
+    whenanswered timestamp with time zone
+);
+
+create unique index analysis_data_message_id_idx
+    on analysis_data(message_id);
+
 -- message_log
 -- Events relating to each message.
 create table message_log (

perllib/FYR.pm

@@ -129,6 +129,7 @@ sub scrub_data ($) {
             state = 'anonymised'
         where id = ?", {}, $id);
     dbh()->do(q#delete from message_log where message_id = ?#, {}, $id);
+    dbh()->do(q#delete from analysis_data where message_id = ?#, {}, $id);
 }
 
 1;

perllib/FYR/Queue.pm

@@ -30,6 +30,7 @@ use FindBin;
 use HTML::Entities;
 use IO::Socket;
 use IO::All;
+use JSON;
 use POSIX qw(strftime);
 use Text::Wrap (); # don't pollute our namespace
 use Time::HiRes ();
@@ -1616,6 +1617,26 @@ sub get_questionnaire_message ($) {
     }
 }
 
+=item record_analysis_data MSGID MSG_SUMMARY ANALYSIS_DATA
+
+Record a user's response to the post confirm questionnaire for some extra analysis data.
+
+MSGID is the messageid
+MSG_SUMMARY is a text string with an optional description of the message,
+ANALYSIS_DATA is a hashref that will be punted as is into the jsonb field.
+
+=cut
+
+sub record_analysis_data($$$) {
+    my ($msgid, $msg_summary, $analysis_data) = @_;
+
+    my $encoded_data = encode_json($analysis_data);
+    dbh()->do('insert into analysis_data (message_id, message_summary, analysis_data, whenanswered) values (?, ?, ?, \'now\') on conflict(message_id) do update set message_summary = ?, analysis_data = ?, whenanswered = \'now\'', {}, $msgid, $msg_summary, $encoded_data, $msg_summary, $encoded_data);
+    dbh()->commit();
+
+    return 1;
+}
+
 #
 # Implementation of the state machine.
 #

perllib/FYR/TestHarness.pm

@@ -161,6 +161,7 @@ sub confirm_message {
     my $cobrand = $options->{cobrand}; 
     my $wth = $options->{wth};
     my $base_url = $options->{base_url};
+    my $do_survey = $options->{do_post_confirm_survey};
 
     my $reptype = $repinfo->{reptype};
     my $repname = $repinfo->{repname};
@@ -197,7 +198,24 @@ sub confirm_message {
         $url =~ s/\s//gs;
         $wth->email_check_url($url);
         $wth->browser_get($url);
-        $wth->browser_check_contents("Your message is on its way.");
+        $wth->browser_check_contents("Your message is on its way");
+        my $summary = "a test message to " . name_n($who);
+        if ($do_survey) {
+            $wth->browser_submit_form(
+                form_name => 'analysisForm',
+                fields => {
+                    msg_summary => $summary,
+                    reason => 'casework',
+                },
+                button => 'submit',
+            );
+            $wth->browser_check_contents('Thanks for your help');
+            my $data = dbh()->selectall_arrayref("select message_id from analysis_data
+                where message_summary = ?", {}, $summary);
+            my $got = scalar @$data;
+            die "No survey results found matching $summary" if ($got == 0);
+            die "Too many survey results" if ($got > 1);
+        }
     }
 }
 

phplib/queue.php

@@ -224,6 +224,17 @@ function msg_record_questionnaire_answer($token, $question, $response) {
     return $result;
 }
 
+/* msg_record_analysis_data ID SUMMARY ANALYSIS_DATA
+
+ Record a user's response to the post confirm option analysis data questionnaire
+*/
+function msg_record_analysis_data($id, $summary, $analysis_data) {
+    global $msg_client;
+    $params = func_get_args();
+    $result = $msg_client->call('FYR.Queue.record_analysis_data', $params);
+    return $result;
+}
+
 /* msg_get_questionnaire_message TOKEN
 
   Return id of the message associated with a questionnaire email. TOKEN is

templates/website/can-you-help.html

@@ -0,0 +1,9 @@
+<div class="card card-lg card-text-center">
+    <?php $values['donate_shown'] = True; ?>
+    <h2>
+    Can you help other people who need to send a message?
+    </h2>
+    <p>With your support, we can help more people get their message to the right place.</p>
+    <a href="https://www.mysociety.org/donate?utm_source=writetothem.com&utm_content=help+others&utm_medium=link&utm_campaign=<?= $values['utm_campaign'] ?>" class="hero__donate__button mysoc-footer__donate__button">Donate now</a>
+</div>
+<br><br>

templates/website/confirm-accept.html

@@ -3,13 +3,14 @@
 $values['title'] = "We’ll send your message now";
 $values['robots'] = 'noindex, nofollow';
 $values['skip_header'] = TRUE;
+$values['utm_campaign'] = 'submitted_page';
 
 template_draw('header', $values);
 
 ?>
 
-<div class="row hero">
-    <div class="large-10 large-centered columns">
+<div class="row hero hero-transparent">
+    <div class="large-8 large-centered columns">
         <div class="ms_header_nav">
             <nav>
                 <ul class="menu">
@@ -19,22 +20,28 @@
         </div>
         <a href="/" class="wtt-logo"><img src="/static/img/logo.png" width="250" height="70" alt="WriteToThem"></a>
         <div class="hero-text-container">
-        <div style="text-align:center;" >
-            <h2>Your message is on its way.</h2>
-            <p class="subtitle">If there is a problem getting your message through, we will email you to tell you. Otherwise, you can sit
-                back and relax.</p>
-        </div>
-        <div style="text-align:center;">
-            <hr style="background-color: white;">
-        </div>
-        <div style="text-align:center;">
-            <?php $values['donate_shown'] = True; ?>
+        <div class="card card-lg">
             <h2>
-            Can you help other people who need to send a message?
+                <svg style="vertical-align: sub;" viewBox="0 0 16 16" width="2.5rem" height="2.5rem" fill="none" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" role="presentation">
+                    <path fill-rule="evenodd" clip-rule="evenodd" d="M8 16C9.05058 16 10.0909 15.7931 11.0615 15.391C12.0321 14.989 12.914 14.3997 13.6569 13.6569C14.3997 12.914 14.989 12.0321 15.391 11.0615C15.7931 10.0909 16 9.05058 16 8C16 6.94943 15.7931 5.90914 15.391 4.93853C14.989 3.96793 14.3997 3.08601 13.6569 2.34315C12.914 1.60028 12.0321 1.011 11.0615 0.608964C10.0909 0.206926 9.05058 -1.56548e-08 8 0C5.87827 3.16163e-08 3.84344 0.842855 2.34315 2.34315C0.842855 3.84344 0 5.87827 0 8C0 10.1217 0.842855 12.1566 2.34315 13.6569C3.84344 15.1571 5.87827 16 8 16ZM7.79378 11.2356L12.2382 5.90222L10.8729 4.76444L7.05067 9.35022L5.07289 7.37156L3.816 8.62844L6.48267 11.2951L7.17067 11.9831L7.79378 11.2356Z" fill="var(--success-color)"/>
+                    </svg>
+                Your message is on its way
             </h2>
-            <p>With your support, we can help more people get their message to the right place.</p>
-            <a href="https://www.mysociety.org/donate?utm_source=writetothem.com&utm_content=help+others&utm_medium=link&utm_campaign=submitted_page" class="hero__donate__button mysoc-footer__donate__button">Donate now</a>
+            <p>If there is a problem getting your message through, we will email you to tell you. Otherwise, you can sit back and relax.</p>
+        </div>
+
+        <br><br>
+        <?php if ($values['form']): ?>
+        <div class="card card-md">
+            <h2 class="subtitle">If you have time could you answer a couple of questions?</h2>
+            <?= $values['form'] ?>
+            <small class="text-muted">We use this information in combination with your postcode to understand what people in different parts of the country are writing to their representatives about. We do not publish the raw responses/postcodes, only summaries of the general topics and areas.</small>
         </div>
+
+        <br><br>
+
+        <?php endif; ?>
+        <?php template_draw('can-you-help', $values) ?>
     </div>
     </div>
 

templates/website/survey-thanks.html

@@ -0,0 +1,38 @@
+<?php
+
+$values['title'] = "Thanks for your help";
+$values['robots'] = 'noindex, nofollow';
+$values['skip_header'] = TRUE;
+$values['utm_campaign'] = 'survey_thanks';
+
+template_draw('header', $values);
+
+?>
+
+<div class="row hero hero-transparent">
+    <div class="large-8 large-centered columns">
+        <div class="ms_header_nav">
+            <nav>
+                <ul class="menu">
+                    <li id="ms_logo"><a class="ms_header_nav-logo" href="https://www.mysociety.org">&nbsp;</a></li>
+                </ul>
+            </nav>
+        </div>
+        <a href="/" class="wtt-logo"><img src="/static/img/logo.png" width="250" height="70" alt="WriteToThem"></a>
+
+        <div class="hero-text-container">
+            <div class="card card-lg">
+                <h2>Thanks for your help</h2>
+                <p class="message">Thank you for telling us more about your message. We'll use this in combination with other people's responses to build a better picture on what people are writing to their representatives about. We do not publish the raw responses/postcodes, only summaries of the general topics and areas.</p>
+            </div>
+
+            <br><br><br>
+            <?php template_draw('can-you-help', $values) ?>
+    </div>
+    </div>
+
+</div>
+
+<?php
+$values['credit'] = '<a href="https://www.freeimages.com/photographer/hisks-57682">Photograph by Krzysztof Szkurlatowski</a>';
+template_draw('footer', $values);

web/confirm.php

@@ -13,18 +13,58 @@
 
 require_once "../phplib/fyr.php";
 require_once "../phplib/queue.php";
+require_once "../phplib/forms.php";
 
 require_once "../commonlib/phplib/utility.php";
 
 fyr_rate_limit(array());
 
+function buildAnalysisForm($values) {
+    global $cobrand, $cocode;
+    $options = cobrand_write_form_options($cobrand);
+    $form_action = cobrand_url($cobrand, '/survey', $cocode);
+    $form = new HTML_QuickForm('analysisForm', 'post', $form_action);
+    $form->setAttribute('class', 'analysis-form');
+    $form->addElement('textarea', 'msg_summary', "Can you tell us in a sentence what your message was about?", array('class' => 'msg-summary'));
+
+    $casework_element = $form->createElement('radio', 'reason', null, '', 'casework');
+    $campaigning_element = $form->createElement('radio', 'reason', null, '', 'campainging');
+
+    $casework_id = $casework_element->getAttribute('id');
+    $campaigning_id = $campaigning_element->getAttribute('id');
+    $legend_id = 'reason_legend';
+
+    // Creates a fieldset with radio options and legend
+    $form->addElement('html', "
+    <fieldset>
+        <legend id=\"{$legend_id}\">Which of the following best describes why you are writing to your representative?</legend>
+        <div class=\"input-card-grid\">
+            <div class=\"radio-card\">
+                <input name=\"reason\" value=\"casework\" type=\"radio\" id=\"{$casework_id}\" aria-labelledby=\"{$legend_id}\">
+                <label for=\"{$casework_id}\">Casework <span class=\"label-hint\">Trying to resolve a problem you or another person is having</span></label>
+            </div>
+            <div class=\"radio-card\">
+                <input name=\"reason\" value=\"campainging\" type=\"radio\" id=\"{$campaigning_id}\" aria-labelledby=\"{$legend_id}\">
+                <label for=\"{$campaigning_id}\">Campaigning<span class=\"label-hint\">Seeking to persuade or inform your representative about a wider issue</span></label>
+            </div>
+        </div>
+    </fieldset>
+    ");
+
+    $form->addElement('html', '<input class="button radius success" name="submit" value="Submit" type="submit">');
+    add_all_variables_hidden($form, $values, $options);
+    $r = new HTML_QuickForm_Renderer_mySociety();
+    $form->accept($r);
+    return $r->toHtml();
+}
 
 $ad = get_http_var('ad');
 if ($ad) {
     $values = array(
         'recipient_via' => null, 'recipient_name' => 'Recipient Name', 'recipient_type' => 'Type',
         'sender_name' => 'Sender Name', 'sender_email' => 'email', 'sender_postcode' => 'SW1A1AA',
-        'group_id' => '', 'advert' => $ad, 'cobrand' => $cobrand, 'host' => fyr_get_host()
+        'group_id' => '', 'advert' => $ad, 'cobrand' => $cobrand, 'host' => fyr_get_host(),
+        'form' => buildAnalysisForm(array()),
     );
     template_draw("confirm-accept", $values);
     exit;
@@ -68,6 +108,7 @@
     } else {
         $values['cobrand'] = $cobrand;
         $values['host'] = fyr_get_host();
+        $values['form'] = buildAnalysisForm(array("msg_id" => $result));
         template_draw("confirm-accept", $values);
     }
 }

web/services/queue.cgi

@@ -61,6 +61,9 @@ while ($req->Accept() >= 0) {
             'FYR.Queue.record_questionnaire_answer' => sub {
                 return FYR::Queue::record_questionnaire_answer($_[0], $_[1], $_[2]);
             },
+            'FYR.Queue.record_analysis_data' => sub {
+                return FYR::Queue::record_analysis_data($_[0], $_[1], $_[2]);
+            },
             'FYR.Queue.get_questionnaire_message' => sub {
                 return FYR::Queue::get_questionnaire_message($_[0]);
             },

web/static/sass/_settings.scss

@@ -72,6 +72,11 @@ $primary-color: #4695b3;
 $alert-color: #c63a3a;
 $success-color: #32b353;
 
+// We can use CSS variables for svg icon on the HTML templates
+:root {
+  --success-color: #{$success-color};
+}
+
 // Make sure border radius matches unless we want it different.
 
 // $global-radius: 3px;