Merge bugstest back to master for final v5.0.4 changesgentoo-5.0.4.1

9 files changed, 158 insertions, 66 deletions

diff --git a/Bugzilla/CGI.pm b/Bugzilla/CGI.pm
index 047ffa757..30c530b8c 100644
--- a/Bugzilla/CGI.pm
+++ b/Bugzilla/CGI.pm
@@ -288,6 +288,69 @@ sub close_standby_message {
     }
 }
 
+our $ALLOW_UNSAFE_RESPONSE = 0;
+# responding to text/plain or text/html is safe
+# responding to any request with a referer header is safe
+# some things need to have unsafe responses (attachment.cgi)
+# everything else should get a 403.
+sub _prevent_unsafe_response {
+    my ($self, $headers) = @_;
+    my $safe_content_type_re = qr{
+        ^ (*COMMIT) # COMMIT makes the regex faster
+                    # by preventing back-tracking. see also perldoc pelre.
+        # application/x-javascript, xml, atom+xml, rdf+xml, xml-dtd, and json
+        (?: application/ (?: x(?: -javascript | ml (?: -dtd )? )
+                           | (?: atom | rdf) \+ xml
+                           | json )
+        # text/csv, text/calendar, text/plain, and text/html
+          | text/ (?: c (?: alendar | sv )
+                    | plain
+                    | html )
+        # used for HTTP push responses
+          | multipart/x-mixed-replace)
+    }sx;
+    my $safe_referer_re = do {
+        # Note that urlbase must end with a /.
+        # It almost certainly does, but let's be extra careful.
+        my $urlbase = correct_urlbase();
+        $urlbase =~ s{/$}{};
+        qr{
+            # Begins with literal urlbase
+            ^ (*COMMIT)
+            \Q$urlbase\E
+            # followed by a slash or end of string
+            (?: /
+              | $ )
+        }sx
+    };
+
+    return if $ALLOW_UNSAFE_RESPONSE;
+
+    if (Bugzilla->usage_mode == USAGE_MODE_BROWSER) {
+        # Safe content types are ones that arn't images.
+        # For now let's assume plain text and html are not valid images.
+        my $content_type         = $headers->{'-type'} // $headers->{'-content_type'} // 'text/html';
+        my $is_safe_content_type = $content_type =~ $safe_content_type_re;
+
+        # Safe referers are ones that begin with the urlbase.
+        my $referer         = $self->referer;
+        my $is_safe_referer = $referer && $referer =~ $safe_referer_re;
+
+        if (!$is_safe_referer && !$is_safe_content_type) {
+            print $self->SUPER::header(-type => 'text/html',  -status => '403 Forbidden');
+            if ($content_type ne 'text/html') {
+                print "Untrusted Referer Header\n";
+                if ($ENV{MOD_PERL}) {
+                    my $r = $self->r;
+                    $r->rflush;
+                    $r->status(200);
+                }
+            }
+            exit;
+        }
+    }
+}
+
 # Override header so we can add the cookies in
 sub header {
     my $self = shift;
@@ -302,6 +365,7 @@ sub header {
     else {
         %headers = @_;
     }
+    $self->_prevent_unsafe_response(\%headers);
 
     if ($self->{'_content_disp'}) {
         $headers{'-content_disposition'} = $self->{'_content_disp'};
diff --git a/Bugzilla/Constants.pm b/Bugzilla/Constants.pm
index 92efe87f5..a6516774b 100644
--- a/Bugzilla/Constants.pm
+++ b/Bugzilla/Constants.pm
@@ -200,7 +200,7 @@ use Memoize;
 # CONSTANTS
 #
 # Bugzilla version
-use constant BUGZILLA_VERSION => "5.0.3+";
+use constant BUGZILLA_VERSION => "5.0.4";
 
 # A base link to the current REST Documentation. We place it here
 # as it will need to be updated to whatever the current release is.
diff --git a/Bugzilla/Install/Requirements.pm b/Bugzilla/Install/Requirements.pm
index a688a0ffa..61496d843 100644
--- a/Bugzilla/Install/Requirements.pm
+++ b/Bugzilla/Install/Requirements.pm
@@ -156,11 +156,6 @@ sub REQUIRED_MODULES {
         version => '1.0.1',
     },
     {
-        package => 'File-Slurp',
-        module  => 'File::Slurp',
-        version => '9999.13',
-    },
-    {
         package => 'JSON-XS',
         module  => 'JSON::XS',
         # 2.0 is the first version that will work with JSON::RPC.
diff --git a/Bugzilla/Search.pm b/Bugzilla/Search.pm
index 646f949f5..0694dd98c 100644
--- a/Bugzilla/Search.pm
+++ b/Bugzilla/Search.pm
@@ -2169,7 +2169,9 @@ sub _timestamp_translate {
     my $value = $args->{value};
     my $dbh = Bugzilla->dbh;
 
-    return if $value !~ /^(?:[\+\-]?\d+[hdwmy]s?|now)$/i;
+    # Force parsing of all dates & times, so that we filter weird values out
+    # from users.
+    #return if $value !~ /^(?:[\+\-]?\d+[hdwmy]s?|now)$/i;
 
     $value = SqlifyDate($value);
     # By default, the time is appended to the date, which we don't always want.
diff --git a/attachment.cgi b/attachment.cgi
index 9566be868..2f04e9594 100755
--- a/attachment.cgi
+++ b/attachment.cgi
@@ -35,6 +35,7 @@ use Encode::MIME::Header; # Required to alter Encode::Encoding{'MIME-Q'}.
 local our $cgi = Bugzilla->cgi;
 local our $template = Bugzilla->template;
 local our $vars = {};
+local $Bugzilla::CGI::ALLOW_UNSAFE_RESPONSE = 1;
 
 # All calls to this script should contain an "action" variable whose
 # value determines what the user wants to do.  The code below checks
diff --git a/contrib/jb2bz.py b/contrib/jb2bz.py
index 85f95423a..caaa0c5e2 100755
--- a/contrib/jb2bz.py
+++ b/contrib/jb2bz.py
@@ -17,8 +17,8 @@ This code requires a recent version of Andy Dustman's MySQLdb interface,
 Share and enjoy.
 """
 
-import rfc822, mimetools, multifile, mimetypes, email.utils
-import sys, re, glob, StringIO, os, stat, time
+import email, mimetypes, email.utils
+import sys, re, glob, os, stat, time
 import MySQLdb, getopt
 
 # mimetypes doesn't include everything we might encounter, yet.
@@ -89,10 +89,24 @@ def process_notes_file(current, fname):
 def process_reply_file(current, fname):
     new_note = {}
     reply = open(fname, "r")
-    msg = rfc822.Message(reply)
-    new_note['text'] = "%s\n%s" % (msg['From'], msg.fp.read())
-    new_note['timestamp'] = email.utils.parsedate_tz(msg['Date'])
-    current["notes"].append(new_note)
+    msg = email.message_from_file(reply)
+
+    # Add any attachments that may have been in a followup or reply
+    msgtype = msg.get_content_maintype()
+    if msgtype == "multipart":
+        for part in msg.walk():
+            new_note = {}
+            if part.get_filename() is None:
+                if part.get_content_type() == "text/plain":
+                    new_note['timestamp'] = time.gmtime(email.utils.mktime_tz(email.utils.parsedate_tz(msg['Date'])))
+                    new_note['text'] = "%s\n%s" % (msg['From'], part.get_payload())
+                    current["notes"].append(new_note)
+            else:
+                maybe_add_attachment(part, current)
+    else:
+        new_note['text'] = "%s\n%s" % (msg['From'], msg.get_payload())
+        new_note['timestamp'] = time.gmtime(email.utils.mktime_tz(email.utils.parsedate_tz(msg['Date'])))
+        current["notes"].append(new_note)
 
 def add_notes(current):
     """Add any notes that have been recorded for the current bug."""
@@ -104,51 +118,48 @@ def add_notes(current):
     for f in glob.glob("%d.followup.*" % current['number']):
         process_reply_file(current, f)
 
-def maybe_add_attachment(current, file, submsg):
+def maybe_add_attachment(submsg, current):
     """Adds the attachment to the current record"""
-    cd = submsg["Content-Disposition"]
-    m = re.search(r'filename="([^"]+)"', cd)
-    if m == None:
+    attachment_filename = submsg.get_filename()
+    if attachment_filename is None:
         return
-    attachment_filename = m.group(1)
-    if (submsg.gettype() == 'application/octet-stream'):
+
+    if (submsg.get_content_type() == 'application/octet-stream'):
         # try get a more specific content-type for this attachment
-        type, encoding = mimetypes.guess_type(m.group(1))
-        if type == None:
-            type = submsg.gettype()
+        mtype, encoding = mimetypes.guess_type(attachment_filename)
+        if mtype == None:
+            mtype = submsg.get_content_type()
     else:
-        type = submsg.gettype()
+        mtype = submsg.get_content_type()
 
-    try:
-        data = StringIO.StringIO()
-        mimetools.decode(file, data, submsg.getencoding())
-    except:
+    if mtype == 'application/x-pkcs7-signature':
+        return
+
+    if mtype == 'application/pkcs7-signature':
+         return
+
+    if mtype == 'application/pgp-signature':
         return
 
-    current['attachments'].append( ( attachment_filename, type, data.getvalue() ) )
+    if mtype == 'message/rfc822':
+        return
 
-def process_mime_body(current, file, submsg):
-    data = StringIO.StringIO()
     try:
-        mimetools.decode(file, data, submsg.getencoding())
-        current['description'] = data.getvalue()
+        data = submsg.get_payload(decode=True)
     except:
         return
 
+    current['attachments'].append( ( attachment_filename, mtype, data ) )
+
 def process_text_plain(msg, current):
-    current['description'] = msg.fp.read()
-
-def process_multi_part(file, msg, current):
-    mf = multifile.MultiFile(file)
-    mf.push(msg.getparam("boundary"))
-    while mf.next():
-        submsg = mimetools.Message(file)
-        if submsg.has_key("Content-Disposition"):
-            maybe_add_attachment(current, mf, submsg)
+    current['description'] = msg.get_payload()
+
+def process_multi_part(msg, current):
+    for part in msg.walk():
+        if part.get_filename() is None:
+            process_text_plain(part, current)
         else:
-            # This is the message body itself (always?), so process
-            # accordingly
-            process_mime_body(current, mf, submsg)
+            maybe_add_attachment(part, current)
 
 def process_jitterbug(filename):
     current = {}
@@ -158,39 +169,37 @@ def process_jitterbug(filename):
     current['description'] = ''
     current['date-reported'] = ()
     current['short-description'] = ''
-    
-    print "Processing: %d" % current['number']
 
-    file = open(filename, "r")
-    create_date = os.fstat(file.fileno())
-    msg = mimetools.Message(file)
+    print "Processing: %d" % current['number']
 
-    msgtype = msg.gettype()
+    mfile = open(filename, "r")
+    create_date = os.fstat(mfile.fileno())
+    msg = email.message_from_file(mfile)
 
-    add_notes(current)
-    current['date-reported'] = email.utils.parsedate_tz(msg['Date'])
+    current['date-reported'] = time.gmtime(email.utils.mktime_tz(email.utils.parsedate_tz(msg['Date'])))
     if current['date-reported'] is None:
        current['date-reported'] = time.gmtime(create_date[stat.ST_MTIME])
 
     if current['date-reported'][0] < 1900:
        current['date-reported'] = time.gmtime(create_date[stat.ST_MTIME])
 
-    if msg.getparam('Subject') is not None: 
+    if msg.has_key('Subject') is not False:
         current['short-description'] = msg['Subject']
     else:
         current['short-description'] = "Unknown"
 
-    if msgtype[:5] == 'text/':
+    msgtype = msg.get_content_maintype()
+    if msgtype == 'text':
         process_text_plain(msg, current)
-    elif msgtype[:5] == 'text':
-        process_text_plain(msg, current)
-    elif msgtype[:10] == "multipart/":
-        process_multi_part(file, msg, current)
+    elif msgtype == "multipart":
+        process_multi_part(msg, current)
     else:
         # Huh? This should never happen.
         print "Unknown content-type: %s" % msgtype
         sys.exit(1)
 
+    add_notes(current)
+
     # At this point we have processed the message: we have all of the notes and
     # attachments stored, so it's time to add things to the database.
     # The schema for JitterBug 2.14 can be found at:
@@ -219,6 +228,7 @@ def process_jitterbug(filename):
     try:
         cursor.execute( "INSERT INTO bugs SET " \
                         "bug_id=%s," \
+                        "priority='---'," \
                         "bug_severity='normal',"  \
                         "bug_status=%s," \
                         "creation_ts=%s,"  \
@@ -242,7 +252,7 @@ def process_jitterbug(filename):
                           version,
                           component,
                           resolution] )
-    
+
         # This is the initial long description associated with the bug report
         cursor.execute( "INSERT INTO longdescs SET " \
                         "bug_id=%s," \
@@ -253,7 +263,7 @@ def process_jitterbug(filename):
                           reporter,
                           time.strftime("%Y-%m-%d %H:%M:%S", current['date-reported'][:9]),
                           current['description'] ] )
-    
+
         # Add whatever notes are associated with this defect
         for n in current['notes']:
                             cursor.execute( "INSERT INTO longdescs SET " \
@@ -265,15 +275,15 @@ def process_jitterbug(filename):
                              reporter,
                              time.strftime("%Y-%m-%d %H:%M:%S", n['timestamp'][:9]),
                              n['text']])
-    
+
         # add attachments associated with this defect
         for a in current['attachments']:
             cursor.execute( "INSERT INTO attachments SET " \
-                            "bug_id=%s, creation_ts=%s, description='', mimetype=%s," \
+                            "bug_id=%s, creation_ts=%s, description=%s, mimetype=%s," \
                             "filename=%s, submitter_id=%s",
                             [ current['number'],
                               time.strftime("%Y-%m-%d %H:%M:%S", current['date-reported'][:9]),
-                              a[1], a[0], reporter ])
+                              a[0], a[1], a[0], reporter ])
             cursor.execute( "INSERT INTO attach_data SET " \
                             "id=LAST_INSERT_ID(), thedata=%s",
                             [ a[2] ])
diff --git a/docs/en/rst/installing/linux.rst b/docs/en/rst/installing/linux.rst
index f40c5e8f0..e2e05c310 100644
--- a/docs/en/rst/installing/linux.rst
+++ b/docs/en/rst/installing/linux.rst
@@ -49,7 +49,7 @@ graphviz patchutils gcc 'perl(Apache2::SizeLimit)' 'perl(Authen::Radius)'
 'perl(Daemon::Generic)' 'perl(Date::Format)' 'perl(DateTime)'
 'perl(DateTime::TimeZone)' 'perl(DBI)' 'perl(Digest::SHA)' 'perl(Email::MIME)'
 'perl(Email::Reply)' 'perl(Email::Sender)' 'perl(Encode)' 'perl(Encode::Detect)'
-'perl(File::MimeInfo::Magic)' 'perl(File::Slurp)' 'perl(GD)' 'perl(GD::Graph)'
+'perl(File::MimeInfo::Magic)' 'perl(GD)' 'perl(GD::Graph)'
 'perl(GD::Text)' 'perl(HTML::FormatText::WithLinks)' 'perl(HTML::Parser)'
 'perl(HTML::Scrubber)' 'perl(IO::Scalar)' 'perl(JSON::RPC)' 'perl(JSON::XS)'
 'perl(List::MoreUtils)' 'perl(LWP::UserAgent)' 'perl(Math::Random::ISAAC)'
diff --git a/docs/en/rst/installing/windows.rst b/docs/en/rst/installing/windows.rst
index e2137a9fc..d756077b0 100644
--- a/docs/en/rst/installing/windows.rst
+++ b/docs/en/rst/installing/windows.rst
@@ -85,7 +85,6 @@ Install the following mandatory modules with:
 * URI
 * List-MoreUtils
 * Math-Random-ISAAC
-* File-Slurp
 * JSON-XS
 * Win32
 * Win32-API
diff --git a/template/en/default/pages/release-notes.html.tmpl b/template/en/default/pages/release-notes.html.tmpl
index c2be24619..b89e3a61d 100644
--- a/template/en/default/pages/release-notes.html.tmpl
+++ b/template/en/default/pages/release-notes.html.tmpl
@@ -43,6 +43,27 @@
 
 <h2 id="point">Updates in this 5.0.x Release</h2>
 
+<h3>5.0.4</h3>
+
+<p>This release fixes one security issue. See the
+  <a href="https://www.bugzilla.org/security/4.4.12/">Security Advisory</a>
+  for details.</p>
+
+<p>This release also contains the following [% terms.bug %] fixes:</p>
+
+<ul>
+  <li><kbd>checksetup.pl</kbd> would fail to update Chart storage during pre-3.6 to 5.0 upgrade.
+    (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1273846">[% terms.Bug %] 1273846</a>)</li>
+  <li><kbd>editflagtypes.cgi</kbd> would crash when classifications are enabled and
+    the user did not have global <kbd>editcomponents</kbd> privileges.
+    (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1310728">[% terms.Bug %] 1310728</a>)</li>
+  <li>The <kbd>File::Slurp</kbd> would trigger warnings on perl 5.24.
+    (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1301887">[% terms.Bug %] 1301887</a>)</li>
+  <li>All the time entries in the 'when' column had the correct date but the time
+    was fixed to 00:00 when using Sqlite.
+    (<a href="https://bugzilla.mozilla.org/show_bug.cgi?id=1303702">[% terms.Bug %] 1303702</a>)</li>
+</ul>
+
 <h3>5.0.3</h3>
 
 <p>This release fixes one security issue. See the
@@ -174,7 +195,7 @@
 <h3 id="req_modules">Required Perl Modules</h3>
 
 [% INCLUDE req_table reqs = REQUIRED_MODULES
-                     new = ['File-Slurp','JSON-XS', 'Email-Sender']
+                     new = ['JSON-XS', 'Email-Sender']
                      updated = ['DateTime', 'DateTime-TimeZone',
                                 'Template-Toolkit', 'URI'] %]