Skip to content

Commit 8b65cac

Browse files
pbailiepbailie
authored
Submitty Autofeed update (#20)
* Update config.php and submitty_student_auto_feed.php * Now can discard a header row, if it exists. * bugfixes in error logging. * Update submitty_student_auto_feed.php WIP * IMAP Adding ability to access IMAP mail server for CSV file. WIP. * JSON remote file helper script Started working on helper script to process remote access JSON data files. * Remote JSON data WIP * Update json_remote.php bugfixes * Update json_remote.php WIP * Update json_remote.php WIP * IMAP WIP * json_remote.php Check for empty data points before writing row to CSV. * Update submitty_student_auto_feed.php Server connection CLI argument -s * Update json_remote.php bugfix * Auto Feed Script WIP * Auto Feed IMAP WIP Preferred first names cannot be upserted once they are set. * Updates to autofeed New IMAP remote script to retrieve datasheets from email and associated config options. * Update imap_remote.php WIP * Update imap_remote.php WIP * Update json_remote.php WIP * Update config.php WIP * ssaf_remote.sh Shell script to help manage use of imap/json_remote.php and submitty_student_auto_feed.php with cron. * Update ssaf_remote.sh WIP * Update imap_remote.php WIP * Updates to auto feed * Update readme.md Remove extraneous `\` noted under json_remote.php * Update config.php Documentation update and spelling fixes. * Update submitty_student_auto_feed.php
1 parent a9d633f commit 8b65cac

File tree

6 files changed

+614
-77
lines changed

6 files changed

+614
-77
lines changed

‎student_auto_feed/config.php

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,14 @@
77
*
88
* Requires minimum PHP version 7.0 with pgsql and iconv extensions.
99
*
10-
* Configuration of submitty_student_auto_feed is structured through defined
11-
* constants. Expanded instructions can be found at
12-
* http://submitty.org/sysadmin/student_auto_feed
10+
* Configuration of submitty_student_auto_feed is structured through a series
11+
* of named constants.
12+
*
13+
* THIS SOFTWARE IS PROVIDED AS IS AND HAS NO GUARANTEE THAT IT IS SAFE OR
14+
* COMPATIBLE WITH YOUR UNIVERSITY'S INFORMATION SYSTEMS. THIS IS ONLY A CODE
15+
* EXAMPLE FOR YOUR UNIVERSITY'S SYSYTEM'S PROGRAMMER TO PROVIDE AN
16+
* IMPLEMENTATION. IT MAY REQUIRE SOME ADDITIONAL MODIFICATION TO SAFELY WORK
17+
* WITH YOUR UNIVERSITY'S AND/OR DEPARTMENT'S INFORMATION SYSTEMS.
1318
*
1419
* -------------------------------------------------------------------------- */
1520

@@ -29,12 +34,13 @@
2934
* -------------------------------------------------------------------------- */
3035

3136
// Univeristy campus's timezone. ***THIS NEEDS TO BE SET.
32-
date_default_timezone_set('America/New_York');
37+
date_default_timezone_set('Etc/UTC');
3338

3439

3540
/* Definitions for error logging -------------------------------------------- */
3641
// While not recommended, email reports of errors may be disabled by setting
37-
// 'ERROR_EMAIL' to null.
42+
// 'ERROR_EMAIL' to null. Please ensure the server running this script has
43+
// sendmail (or equivalent) installed. Email is sent "unauthenticated".
3844
define('ERROR_EMAIL', '[email protected]');
3945
define('ERROR_LOG_FILE', '/var/local/submitty/bin/auto_feed_error.log');
4046

@@ -74,9 +80,11 @@
7480
define('CSV_DELIM_CHAR', chr(9));
7581

7682
//Properties for database access. ***THESE NEED TO BE SET.
83+
//If multiple instances of Submitty are being supported, these may be defined as
84+
//parrallel arrays.
7785
define('DB_HOST', 'submitty.cs.myuniversity.edu');
78-
define('DB_LOGIN', 'hsdbu');
79-
define('DB_PASSWORD', 'DB.p4ssw0rd');
86+
define('DB_LOGIN', 'my_database_user'); //DO NOT USE IN PRODUCTION
87+
define('DB_PASSWORD', 'my_database_password'); //DO NOT USE IN PRODUCTION
8088

8189
/* The following constants identify what columns to read in the CSV dump. --- */
8290
//these properties are used to group data by individual course and student.
@@ -97,6 +105,43 @@
97105
//Validate term code. Set to null to disable this check.
98106
define('EXPECTED_TERM_CODE', '201705');
99107

108+
//Header row, if it exists, must be discarded during processing.
109+
define('HEADER_ROW_EXISTS', true);
110+
111+
//Remote IMAP
112+
//This is used by imap_remote.php to login and retrieve a student enrollment
113+
//datasheet, should datasheets be provided via an IMAP email box. This also
114+
//works with exchange servers with IMAP enabled.
115+
//IMAP_FOLDER is the folder where the data sheets can be found.
116+
//IMAP_OPTIONS: q.v. "Optional flags for names" at https://www.php.net/manual/en/function.imap-open.php
117+
//IMAP_FROM is for validation. Make sure it matches the identity of who sends the data sheets
118+
//IMAP_SUBJECT is for validation. Make sure it matches the subject line of the messages containing the data sheet.
119+
//IMAP_ATTACHMENT is for validation. Make sure it matches the file name of the attached data sheets.
120+
define('IMAP_HOSTNAME', 'imap.cs.myuniversity.edu');
121+
define('IMAP_PORT', '993');
122+
define('IMAP_USERNAME', 'imap_user'); //DO NOT USE IN PRODUCTION
123+
define('IMAP_PASSWORD', 'imap_password'); //DO NOT USE IN PRODUCTION
124+
define('IMAP_FOLDER', 'INBOX');
125+
define('IMAP_OPTIONS', array('imap', 'ssl'));
126+
define('IMAP_FROM', 'Data Warehouse');
127+
define('IMAP_SUBJECT', 'Your daily CSV');
128+
define('IMAP_ATTACHMENT', 'submitty_enrollments.csv');
129+
130+
//Remote JSON
131+
//This is used by json_remote.php to read JSON data from another server via
132+
//an SSH session. The JSON data is then written to a CSV file usable by the
133+
//auto feed.
134+
//JSON_REMOTE_FINGERPRINT must match the SSH fingerprint of the server being
135+
//accessed. This is to help ensure you are not connecting to an imposter server,
136+
//such as with a man-in-the-middle attack.
137+
//JSON_REMOTE_PATH is the remote path to the JSON data file(s).
138+
define('JSON_REMOTE_HOSTNAME', 'server.cs.myuniversity.edu');
139+
define('JSON_REMOTE_PORT', 22);
140+
define('JSON_REMOTE_FINGERPRINT', '00112233445566778899AABBCCDDEEFF00112233');
141+
define('JSON_REMOTE_USERNAME', 'json_user'); //DO NOT USE IN PRODUCTION
142+
define('JSON_REMOTE_PASSWORD', 'json_password'); //DO NOT USE IN PRODUCTION
143+
define('JSON_REMOTE_PATH', '/path/to/files/');
144+
100145
//Sometimes data feeds are generated by Windows systems, in which case the data
101146
//file probably needs to be converted from Windows-1252 (aka CP-1252) to UTF-8.
102147
//Set to true to convert data feed file from Windows char encoding to UTF-8.
@@ -106,4 +151,5 @@
106151
//Allows "\r" EOL encoding. This is rare but exists (e.g. Excel for Macintosh).
107152
ini_set('auto_detect_line_endings', true);
108153

154+
//EOF
109155
?>

‎student_auto_feed/imap_remote.php

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/usr/bin/env php
2+
<?php
3+
/**
4+
* Helper script for retrieving CSV datasheets from IMAP email account messages.
5+
*
6+
* The student auto feed script is designed to read a CSV export of student
7+
* enrollment. This helper script is intended to read the CSV export from an
8+
* IMAP email account message attachment and write the data to as a local file.
9+
* Requires PHP 7.0+ with imap library.
10+
*
11+
* @author Peter Bailie, Rensselaer Polytechnic Institute
12+
*/
13+
require "config.php";
14+
15+
new imap_remote();
16+
exit(0);
17+
18+
/** Class to retrieve CSV datasheet from IMAP and write it to filesystem */
19+
class imap_remote {
20+
21+
/** @static @property resource */
22+
private static $imap_conn;
23+
24+
/** @static @property resource */
25+
private static $csv_fh;
26+
27+
/** @static @property boolean */
28+
private static $csv_locked = false;
29+
30+
public function __construct() {
31+
switch(false) {
32+
case $this->imap_connect():
33+
exit(1);
34+
case $this->get_csv_data():
35+
exit(1);
36+
}
37+
}
38+
39+
public function __destruct() {
40+
$this->close_csv();
41+
$this->imap_disconnect();
42+
}
43+
44+
/**
45+
* Open connection to IMAP server.
46+
*
47+
* @access private
48+
* @return boolean true when connection established, false otherwise.
49+
*/
50+
private function imap_connect() {
51+
//gracefully close any existing imap connections (shouldn't be any, but just in case...)
52+
$this->imap_disconnect();
53+
54+
$hostname = IMAP_HOSTNAME;
55+
$port = IMAP_PORT;
56+
$username = IMAP_USERNAME;
57+
$password = IMAP_PASSWORD;
58+
$msg_folder = IMAP_FOLDER;
59+
$options = "/" . implode("/", IMAP_OPTIONS);
60+
$auth = "{{$hostname}:{$port}{$options}}{$msg_folder}";
61+
62+
self::$imap_conn = imap_open($auth, $username, $password, null, 3, array('DISABLE_AUTHENTICATOR' => 'GSSAPI'));
63+
64+
if (is_resource(self::$imap_conn) && get_resource_type(self::$imap_conn) === "imap") {
65+
return true;
66+
} else {
67+
fprintf(STDERR, "Cannot connect to {$hostname}.\n%s\n", imap_last_error());
68+
return false;
69+
}
70+
}
71+
72+
/**
73+
* Close connection to IMAP server.
74+
*
75+
* @access private
76+
*/
77+
private function imap_disconnect() {
78+
if (is_resource(self::$imap_conn) && get_resource_type(self::$imap_conn) === "imap") {
79+
imap_close(self::$imap_conn);
80+
}
81+
}
82+
83+
/**
84+
* Open/lock CSV file for writing.
85+
*
86+
* @access private
87+
* @return boolean true on success, false otherwise
88+
*/
89+
private function open_csv() {
90+
//gracefully close any open file handles (shouldn't be any, but just in case...)
91+
$this->close_csv();
92+
93+
//Open CSV for writing.
94+
self::$csv_fh = fopen(CSV_FILE, "w");
95+
if (!is_resource(self::$csv_fh) || get_resource_type(self::$csv_fh) !== "stream") {
96+
fprintf(STDERR, "Could not open CSV file for writing.\n%s\n", error_get_last());
97+
return false;
98+
}
99+
100+
//Lock CSV file.
101+
if (flock(self::$csv_fh, LOCK_SH, $wouldblock)) {
102+
self::$csv_locked = true;
103+
return true;
104+
} else if ($wouldblock === 1) {
105+
fprintf(STDERR, "Another process has locked the CSV.\n%s\n", error_get_last());
106+
return false;
107+
} else {
108+
fprintf(STDERR, "CSV not blocked, but still could not attain lock for writing.\n%s\n", error_get_last());
109+
return false;
110+
}
111+
}
112+
113+
/**
114+
* Close/Unlock CSV file from writing.
115+
*
116+
* @access private
117+
*/
118+
private function close_csv() {
119+
//Unlock CSV file, if it is locked.
120+
if (self::$csv_locked && flock(self::$csv_fh, LOCK_UN)) {
121+
self::$csv_locked = false;
122+
}
123+
124+
//Close CSV file, if it is open.
125+
if (is_resource(self::$csv_fh) && get_resource_type(self::$csv_fh) === "stream") {
126+
fclose(self::$csv_fh);
127+
}
128+
}
129+
130+
/**
131+
* Get CSV attachment and write it to a file.
132+
*
133+
* @access private
134+
* @return boolean true on success, false otherwise.
135+
*/
136+
private function get_csv_data() {
137+
$imap_from = IMAP_FROM;
138+
$imap_subject = IMAP_SUBJECT;
139+
$search_string = "UNSEEN FROM \"{$imap_from}\" SUBJECT \"{$imap_subject}\"";
140+
$email_id = imap_search(self::$imap_conn, $search_string);
141+
142+
//Should only be one message to process.
143+
if (!is_array($email_id) || count($email_id) != 1) {
144+
fprintf(STDERR, "Expected one valid datasheet via IMAP mail.\nMessage IDs found (\"false\" means none):\n%s\n", var_export($email_id, true));
145+
return false;
146+
}
147+
148+
//Open CSV for writing.
149+
if (!$this->open_csv()) {
150+
return false;
151+
}
152+
153+
//Locate file attachment via email structure parts.
154+
$structure = imap_fetchstructure(self::$imap_conn, $email_id[0]);
155+
foreach($structure->parts as $part_index=>$part) {
156+
//Is there an attachment?
157+
if ($part->ifdisposition === 1 && $part->disposition === "attachment") {
158+
159+
//Scan through email structure and validate attachment.
160+
$ifparams_list = array($part->ifdparameters, $part->ifparameters); //indicates if (d)paramaters exist.
161+
$params_list = array($part->dparameters, $part->parameters); //(d)parameter data, parrallel array to $ifparams_list.
162+
foreach($ifparams_list as $ifparam_index=>$ifparams) {
163+
if ((boolean)$ifparams) {
164+
foreach($params_list[$ifparam_index] as $params) {
165+
if (strpos($params->attribute, "name") !== false && $params->value === IMAP_ATTACHMENT) {
166+
//Get attachment data.
167+
switch($part->encoding) {
168+
//7 bit is ASCII. 8 bit is Latin-1. Both should be printable without decoding.
169+
case ENC7BIT:
170+
case ENC8BIT:
171+
fwrite(self::$csv_fh, imap_fetchbody(self::$imap_conn, $email_id[0], $part_index+1));
172+
//Set SEEN flag on email so it isn't re-read again in the future.
173+
imap_setflag_full(self::$imap_conn, (string)$email_id[0], "\SEEN");
174+
return true;
175+
//Base64 needs decoding.
176+
case ENCBASE64:
177+
fwrite(self::$csv_fh, imap_base64(imap_fetchbody(self::$imap_conn, $email_id[0], $part_index+1)));
178+
//Set SEEN flag on email so it isn't re-read again in the future.
179+
imap_setflag_full(self::$imap_conn, (string)$email_id[0], "\SEEN");
180+
return true;
181+
//Quoted Printable needs decoding.
182+
case ENCQUOTEDPRINTABLE:
183+
fwrite(self::$csv_fh, imap_qprint(imap_fetchbody(self::$imap_conn, $email_id[0], $part_index+1)));
184+
//Set SEEN flag on email so it isn't re-read again in the future.
185+
imap_setflag_full(self::$imap_conn, (string)$email_id[0], "\SEEN");
186+
return true;
187+
default:
188+
fprintf(STDERR, "Unexpected character encoding: %s\n(2 = BINARY, 5 = OTHER)\n", $part->encoding);
189+
break;
190+
}
191+
}
192+
}
193+
}
194+
}
195+
}
196+
}
197+
198+
// If we're down here, something has gone wrong.
199+
fprintf(STDERR, "Unexpected error while trying to write CSV.\n%s\n", error_get_last());
200+
return false;
201+
}
202+
} //END class imap
203+
?>

0 commit comments

Comments
 (0)