lib/codereview/codereview.py

   1 # coding=utf-8
   2 # (The line above is necessary so that I can use 世界 in the
   3 # *comment* below without Python getting all bent out of shape.)
   4
   5 # Copyright 2007-2009 Google Inc.
   6 #
   7 # Licensed under the Apache License, Version 2.0 (the "License");
   8 # you may not use this file except in compliance with the License.
   9 # You may obtain a copy of the License at
  10 #
  11 #       http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18
  19 '''Mercurial interface to codereview.appspot.com.
  20
  21 To configure, set the following options in
  22 your repository's .hg/hgrc file.
  23
  24         [extensions]
  25         codereview = /path/to/codereview.py
  26
  27         [codereview]
  28         server = codereview.appspot.com
  29
  30 The server should be running Rietveld; see http://code.google.com/p/rietveld/.
  31
  32 In addition to the new commands, this extension introduces
  33 the file pattern syntax @nnnnnn, where nnnnnn is a change list
  34 number, to mean the files included in that change list, which
  35 must be associated with the current client.
  36
  37 For example, if change 123456 contains the files x.go and y.go,
  38 "hg diff @123456" is equivalent to"hg diff x.go y.go".
  39 '''
  40
  41 import sys
  42
  43 if __name__ == "__main__":
  44         print >>sys.stderr, "This is a Mercurial extension and should not be invoked directly."
  45         sys.exit(2)
  46
  47 # We require Python 2.6 for the json package.
  48 if sys.version < '2.6':
  49         print >>sys.stderr, "The codereview extension requires Python 2.6 or newer."
  50         print >>sys.stderr, "You are running Python " + sys.version
  51         sys.exit(2)
  52
  53 import json
  54 import os
  55 import re
  56 import stat
  57 import subprocess
  58 import threading
  59 import time
  60
  61 from mercurial import commands as hg_commands
  62 from mercurial import util as hg_util
  63
  64 # bind Plan 9 preferred dotfile location
  65 if os.sys.platform == 'plan9':
  66         try:
  67                 import plan9
  68                 n = plan9.bind(os.path.expanduser("~/lib"), os.path.expanduser("~"), plan9.MBEFORE|plan9.MCREATE)
  69         except ImportError:
  70                 pass
  71
  72 defaultcc = None
  73 codereview_disabled = None
  74 real_rollback = None
  75 releaseBranch = None
  76 server = "codereview.appspot.com"
  77 server_url_base = None
  78 testing = None
  79
  80 #######################################################################
  81 # Normally I would split this into multiple files, but it simplifies
  82 # import path headaches to keep it all in one file.  Sorry.
  83 # The different parts of the file are separated by banners like this one.
  84
  85 #######################################################################
  86 # Helpers
  87
  88 def RelativePath(path, cwd):
  89         n = len(cwd)
  90         if path.startswith(cwd) and path[n] == '/':
  91                 return path[n+1:]
  92         return path
  93
  94 def Sub(l1, l2):
  95         return [l for l in l1 if l not in l2]
  96
  97 def Add(l1, l2):
  98         l = l1 + Sub(l2, l1)
  99         l.sort()
 100         return l
 101
 102 def Intersect(l1, l2):
 103         return [l for l in l1 if l in l2]
 104
 105 #######################################################################
 106 # RE: UNICODE STRING HANDLING
 107 #
 108 # Python distinguishes between the str (string of bytes)
 109 # and unicode (string of code points) types.  Most operations
 110 # work on either one just fine, but some (like regexp matching)
 111 # require unicode, and others (like write) require str.
 112 #
 113 # As befits the language, Python hides the distinction between
 114 # unicode and str by converting between them silently, but
 115 # *only* if all the bytes/code points involved are 7-bit ASCII.
 116 # This means that if you're not careful, your program works
 117 # fine on "hello, world" and fails on "hello, 世界".  And of course,
 118 # the obvious way to be careful - use static types - is unavailable.
 119 # So the only way is trial and error to find where to put explicit
 120 # conversions.
 121 #
 122 # Because more functions do implicit conversion to str (string of bytes)
 123 # than do implicit conversion to unicode (string of code points),
 124 # the convention in this module is to represent all text as str,
 125 # converting to unicode only when calling a unicode-only function
 126 # and then converting back to str as soon as possible.
 127
 128 def typecheck(s, t):
 129         if type(s) != t:
 130                 raise hg_util.Abort("type check failed: %s has type %s != %s" % (repr(s), type(s), t))
 131
 132 # If we have to pass unicode instead of str, ustr does that conversion clearly.
 133 def ustr(s):
 134         typecheck(s, str)
 135         return s.decode("utf-8")
 136
 137 # Even with those, Mercurial still sometimes turns unicode into str
 138 # and then tries to use it as ascii.  Change Mercurial's default.
 139 def set_mercurial_encoding_to_utf8():
 140         from mercurial import encoding
 141         encoding.encoding = 'utf-8'
 142
 143 set_mercurial_encoding_to_utf8()
 144
 145 # Even with those we still run into problems.
 146 # I tried to do things by the book but could not convince
 147 # Mercurial to let me check in a change with UTF-8 in the
 148 # CL description or author field, no matter how many conversions
 149 # between str and unicode I inserted and despite changing the
 150 # default encoding.  I'm tired of this game, so set the default
 151 # encoding for all of Python to 'utf-8', not 'ascii'.
 152 def default_to_utf8():
 153         import sys
 154         stdout, __stdout__ = sys.stdout, sys.__stdout__
 155         reload(sys)  # site.py deleted setdefaultencoding; get it back
 156         sys.stdout, sys.__stdout__ = stdout, __stdout__
 157         sys.setdefaultencoding('utf-8')
 158
 159 default_to_utf8()
 160
 161 #######################################################################
 162 # Status printer for long-running commands
 163
 164 global_status = None
 165
 166 def set_status(s):
 167         if verbosity > 0:
 168                 print >>sys.stderr, time.asctime(), s
 169         global global_status
 170         global_status = s
 171
 172 class StatusThread(threading.Thread):
 173         def __init__(self):
 174                 threading.Thread.__init__(self)
 175         def run(self):
 176                 # pause a reasonable amount of time before
 177                 # starting to display status messages, so that
 178                 # most hg commands won't ever see them.
 179                 time.sleep(30)
 180
 181                 # now show status every 15 seconds
 182                 while True:
 183                         time.sleep(15 - time.time() % 15)
 184                         s = global_status
 185                         if s is None:
 186                                 continue
 187                         if s == "":
 188                                 s = "(unknown status)"
 189                         print >>sys.stderr, time.asctime(), s
 190
 191 def start_status_thread():
 192         t = StatusThread()
 193         t.setDaemon(True)  # allowed to exit if t is still running
 194         t.start()
 195
 196 #######################################################################
 197 # Change list parsing.
 198 #
 199 # Change lists are stored in .hg/codereview/cl.nnnnnn
 200 # where nnnnnn is the number assigned by the code review server.
 201 # Most data about a change list is stored on the code review server
 202 # too: the description, reviewer, and cc list are all stored there.
 203 # The only thing in the cl.nnnnnn file is the list of relevant files.
 204 # Also, the existence of the cl.nnnnnn file marks this repository
 205 # as the one where the change list lives.
 206
 207 emptydiff = """Index: ~rietveld~placeholder~
 208 ===================================================================
 209 diff --git a/~rietveld~placeholder~ b/~rietveld~placeholder~
 210 new file mode 100644
 211 """
 212
 213 class CL(object):
 214         def __init__(self, name):
 215                 typecheck(name, str)
 216                 self.name = name
 217                 self.desc = ''
 218                 self.files = []
 219                 self.reviewer = []
 220                 self.cc = []
 221                 self.url = ''
 222                 self.local = False
 223                 self.web = False
 224                 self.copied_from = None # None means current user
 225                 self.mailed = False
 226                 self.private = False
 227                 self.lgtm = []
 228
 229         def DiskText(self):
 230                 cl = self
 231                 s = ""
 232                 if cl.copied_from:
 233                         s += "Author: " + cl.copied_from + "\n\n"
 234                 if cl.private:
 235                         s += "Private: " + str(self.private) + "\n"
 236                 s += "Mailed: " + str(self.mailed) + "\n"
 237                 s += "Description:\n"
 238                 s += Indent(cl.desc, "\t")
 239                 s += "Files:\n"
 240                 for f in cl.files:
 241                         s += "\t" + f + "\n"
 242                 typecheck(s, str)
 243                 return s
 244
 245         def EditorText(self):
 246                 cl = self
 247                 s = _change_prolog
 248                 s += "\n"
 249                 if cl.copied_from:
 250                         s += "Author: " + cl.copied_from + "\n"
 251                 if cl.url != '':
 252                         s += 'URL: ' + cl.url + '       # cannot edit\n\n'
 253                 if cl.private:
 254                         s += "Private: True\n"
 255                 s += "Reviewer: " + JoinComma(cl.reviewer) + "\n"
 256                 s += "CC: " + JoinComma(cl.cc) + "\n"
 257                 s += "\n"
 258                 s += "Description:\n"
 259                 if cl.desc == '':
 260                         s += "\t<enter description here>\n"
 261                 else:
 262                         s += Indent(cl.desc, "\t")
 263                 s += "\n"
 264                 if cl.local or cl.name == "new":
 265                         s += "Files:\n"
 266                         for f in cl.files:
 267                                 s += "\t" + f + "\n"
 268                         s += "\n"
 269                 typecheck(s, str)
 270                 return s
 271
 272         def PendingText(self, quick=False):
 273                 cl = self
 274                 s = cl.name + ":" + "\n"
 275                 s += Indent(cl.desc, "\t")
 276                 s += "\n"
 277                 if cl.copied_from:
 278                         s += "\tAuthor: " + cl.copied_from + "\n"
 279                 if not quick:
 280                         s += "\tReviewer: " + JoinComma(cl.reviewer) + "\n"
 281                         for (who, line, _) in cl.lgtm:
 282                                 s += "\t\t" + who + ": " + line + "\n"
 283                         s += "\tCC: " + JoinComma(cl.cc) + "\n"
 284                 s += "\tFiles:\n"
 285                 for f in cl.files:
 286                         s += "\t\t" + f + "\n"
 287                 typecheck(s, str)
 288                 return s
 289
 290         def Flush(self, ui, repo):
 291                 if self.name == "new":
 292                         self.Upload(ui, repo, gofmt_just_warn=True, creating=True)
 293                 dir = CodeReviewDir(ui, repo)
 294                 path = dir + '/cl.' + self.name
 295                 f = open(path+'!', "w")
 296                 f.write(self.DiskText())
 297                 f.close()
 298                 if sys.platform == "win32" and os.path.isfile(path):
 299                         os.remove(path)
 300                 os.rename(path+'!', path)
 301                 if self.web and not self.copied_from:
 302                         EditDesc(self.name, desc=self.desc,
 303                                 reviewers=JoinComma(self.reviewer), cc=JoinComma(self.cc),
 304                                 private=self.private)
 305
 306         def Delete(self, ui, repo):
 307                 dir = CodeReviewDir(ui, repo)
 308                 os.unlink(dir + "/cl." + self.name)
 309
 310         def Subject(self, ui, repo):
 311                 s = line1(self.desc)
 312                 if len(s) > 60:
 313                         s = s[0:55] + "..."
 314                 if self.name != "new":
 315                         s = "code review %s: %s" % (self.name, s)
 316                 typecheck(s, str)
 317                 return branch_prefix(ui, repo) + s
 318
 319         def Upload(self, ui, repo, send_mail=False, gofmt=True, gofmt_just_warn=False, creating=False, quiet=False):
 320                 if not self.files and not creating:
 321                         ui.warn("no files in change list\n")
 322                 if ui.configbool("codereview", "force_gofmt", True) and gofmt:
 323                         CheckFormat(ui, repo, self.files, just_warn=gofmt_just_warn)
 324                 set_status("uploading CL metadata + diffs")
 325                 os.chdir(repo.root)
 326
 327                 form_fields = [
 328                         ("content_upload", "1"),
 329                         ("reviewers", JoinComma(self.reviewer)),
 330                         ("cc", JoinComma(self.cc)),
 331                         ("description", self.desc),
 332                         ("base_hashes", ""),
 333                 ]
 334
 335                 if self.name != "new":
 336                         form_fields.append(("issue", self.name))
 337                 vcs = None
 338                 # We do not include files when creating the issue,
 339                 # because we want the patch sets to record the repository
 340                 # and base revision they are diffs against.  We use the patch
 341                 # set message for that purpose, but there is no message with
 342                 # the first patch set.  Instead the message gets used as the
 343                 # new CL's overall subject.  So omit the diffs when creating
 344                 # and then we'll run an immediate upload.
 345                 # This has the effect that every CL begins with an empty "Patch set 1".
 346                 if self.files and not creating:
 347                         vcs = MercurialVCS(upload_options, ui, repo)
 348                         data = vcs.GenerateDiff(self.files)
 349                         files = vcs.GetBaseFiles(data)
 350                         if len(data) > MAX_UPLOAD_SIZE:
 351                                 uploaded_diff_file = []
 352                                 form_fields.append(("separate_patches", "1"))
 353                         else:
 354                                 uploaded_diff_file = [("data", "data.diff", data)]
 355                 else:
 356                         uploaded_diff_file = [("data", "data.diff", emptydiff)]
 357
 358                 if vcs and self.name != "new":
 359                         form_fields.append(("subject", "diff -r " + vcs.base_rev + " " + ui.expandpath("default")))
 360                 else:
 361                         # First upload sets the subject for the CL itself.
 362                         form_fields.append(("subject", self.Subject(ui, repo)))
 363
 364                 ctype, body = EncodeMultipartFormData(form_fields, uploaded_diff_file)
 365                 response_body = MySend("/upload", body, content_type=ctype)
 366                 patchset = None
 367                 msg = response_body
 368                 lines = msg.splitlines()
 369                 if len(lines) >= 2:
 370                         msg = lines[0]
 371                         patchset = lines[1].strip()
 372                         patches = [x.split(" ", 1) for x in lines[2:]]
 373                 else:
 374                         print >>sys.stderr, "Server says there is nothing to upload (probably wrong):\n" + msg
 375                 if response_body.startswith("Issue updated.") and quiet:
 376                         pass
 377                 else:
 378                         ui.status(msg + "\n")
 379                 set_status("uploaded CL metadata + diffs")
 380                 if not response_body.startswith("Issue created.") and not response_body.startswith("Issue updated."):
 381                         raise hg_util.Abort("failed to update issue: " + response_body)
 382                 issue = msg[msg.rfind("/")+1:]
 383                 self.name = issue
 384                 if not self.url:
 385                         self.url = server_url_base + self.name
 386                 if not uploaded_diff_file:
 387                         set_status("uploading patches")
 388                         patches = UploadSeparatePatches(issue, rpc, patchset, data, upload_options)
 389                 if vcs:
 390                         set_status("uploading base files")
 391                         vcs.UploadBaseFiles(issue, rpc, patches, patchset, upload_options, files)
 392                 if patchset != "1":
 393                         MySend("/" + issue + "/upload_complete/" + patchset, payload="")
 394                 if send_mail:
 395                         set_status("sending mail")
 396                         MySend("/" + issue + "/mail", payload="")
 397                 self.web = True
 398                 set_status("flushing changes to disk")
 399                 self.Flush(ui, repo)
 400                 return
 401
 402         def Mail(self, ui, repo):
 403                 pmsg = "Hello " + JoinComma(self.reviewer)
 404                 if self.cc:
 405                         pmsg += " (cc: %s)" % (', '.join(self.cc),)
 406                 pmsg += ",\n"
 407                 pmsg += "\n"
 408                 repourl = ui.expandpath("default")
 409                 if not self.mailed:
 410                         pmsg += "I'd like you to review this change to"
 411                         branch = repo[None].branch()
 412                         if branch.startswith("dev."):
 413                                 pmsg += " the " + branch + " branch of"
 414                         pmsg += "\n" + repourl + "\n"
 415                 else:
 416                         pmsg += "Please take another look.\n"
 417                 typecheck(pmsg, str)
 418                 PostMessage(ui, self.name, pmsg, subject=self.Subject(ui, repo))
 419                 self.mailed = True
 420                 self.Flush(ui, repo)
 421
 422 def GoodCLName(name):
 423         typecheck(name, str)
 424         return re.match("^[0-9]+$", name)
 425
 426 def ParseCL(text, name):
 427         typecheck(text, str)
 428         typecheck(name, str)
 429         sname = None
 430         lineno = 0
 431         sections = {
 432                 'Author': '',
 433                 'Description': '',
 434                 'Files': '',
 435                 'URL': '',
 436                 'Reviewer': '',
 437                 'CC': '',
 438                 'Mailed': '',
 439                 'Private': '',
 440         }
 441         for line in text.split('\n'):
 442                 lineno += 1
 443                 line = line.rstrip()
 444                 if line != '' and line[0] == '#':
 445                         continue
 446                 if line == '' or line[0] == ' ' or line[0] == '\t':
 447                         if sname == None and line != '':
 448                                 return None, lineno, 'text outside section'
 449                         if sname != None:
 450                                 sections[sname] += line + '\n'
 451                         continue
 452                 p = line.find(':')
 453                 if p >= 0:
 454                         s, val = line[:p].strip(), line[p+1:].strip()
 455                         if s in sections:
 456                                 sname = s
 457                                 if val != '':
 458                                         sections[sname] += val + '\n'
 459                                 continue
 460                 return None, lineno, 'malformed section header'
 461
 462         for k in sections:
 463                 sections[k] = StripCommon(sections[k]).rstrip()
 464
 465         cl = CL(name)
 466         if sections['Author']:
 467                 cl.copied_from = sections['Author']
 468         cl.desc = sections['Description']
 469         for line in sections['Files'].split('\n'):
 470                 i = line.find('#')
 471                 if i >= 0:
 472                         line = line[0:i].rstrip()
 473                 line = line.strip()
 474                 if line == '':
 475                         continue
 476                 cl.files.append(line)
 477         cl.reviewer = SplitCommaSpace(sections['Reviewer'])
 478         cl.cc = SplitCommaSpace(sections['CC'])
 479         cl.url = sections['URL']
 480         if sections['Mailed'] != 'False':
 481                 # Odd default, but avoids spurious mailings when
 482                 # reading old CLs that do not have a Mailed: line.
 483                 # CLs created with this update will always have
 484                 # Mailed: False on disk.
 485                 cl.mailed = True
 486         if sections['Private'] in ('True', 'true', 'Yes', 'yes'):
 487                 cl.private = True
 488         if cl.desc == '<enter description here>':
 489                 cl.desc = ''
 490         return cl, 0, ''
 491
 492 def SplitCommaSpace(s):
 493         typecheck(s, str)
 494         s = s.strip()
 495         if s == "":
 496                 return []
 497         return re.split(", *", s)
 498
 499 def CutDomain(s):
 500         typecheck(s, str)
 501         i = s.find('@')
 502         if i >= 0:
 503                 s = s[0:i]
 504         return s
 505
 506 def JoinComma(l):
 507         seen = {}
 508         uniq = []
 509         for s in l:
 510                 typecheck(s, str)
 511                 if s not in seen:
 512                         seen[s] = True
 513                         uniq.append(s)
 514
 515         return ", ".join(uniq)
 516
 517 def ExceptionDetail():
 518         s = str(sys.exc_info()[0])
 519         if s.startswith("<type '") and s.endswith("'>"):
 520                 s = s[7:-2]
 521         elif s.startswith("<class '") and s.endswith("'>"):
 522                 s = s[8:-2]
 523         arg = str(sys.exc_info()[1])
 524         if len(arg) > 0:
 525                 s += ": " + arg
 526         return s
 527
 528 def IsLocalCL(ui, repo, name):
 529         return GoodCLName(name) and os.access(CodeReviewDir(ui, repo) + "/cl." + name, 0)
 530
 531 # Load CL from disk and/or the web.
 532 def LoadCL(ui, repo, name, web=True):
 533         typecheck(name, str)
 534         set_status("loading CL " + name)
 535         if not GoodCLName(name):
 536                 return None, "invalid CL name"
 537         dir = CodeReviewDir(ui, repo)
 538         path = dir + "cl." + name
 539         if os.access(path, 0):
 540                 ff = open(path)
 541                 text = ff.read()
 542                 ff.close()
 543                 cl, lineno, err = ParseCL(text, name)
 544                 if err != "":
 545                         return None, "malformed CL data: "+err
 546                 cl.local = True
 547         else:
 548                 cl = CL(name)
 549         if web:
 550                 set_status("getting issue metadata from web")
 551                 d = JSONGet(ui, "/api/" + name + "?messages=true")
 552                 set_status(None)
 553                 if d is None:
 554                         return None, "cannot load CL %s from server" % (name,)
 555                 if 'owner_email' not in d or 'issue' not in d or str(d['issue']) != name:
 556                         return None, "malformed response loading CL data from code review server"
 557                 cl.dict = d
 558                 cl.reviewer = d.get('reviewers', [])
 559                 cl.cc = d.get('cc', [])
 560                 if cl.local and cl.copied_from and cl.desc:
 561                         # local copy of CL written by someone else
 562                         # and we saved a description.  use that one,
 563                         # so that committers can edit the description
 564                         # before doing hg submit.
 565                         pass
 566                 else:
 567                         cl.desc = d.get('description', "")
 568                 cl.url = server_url_base + name
 569                 cl.web = True
 570                 cl.private = d.get('private', False) != False
 571                 cl.lgtm = []
 572                 for m in d.get('messages', []):
 573                         if m.get('approval', False) == True or m.get('disapproval', False) == True:
 574                                 who = re.sub('@.*', '', m.get('sender', ''))
 575                                 text = re.sub("\n(.|\n)*", '', m.get('text', ''))
 576                                 cl.lgtm.append((who, text, m.get('approval', False)))
 577
 578         set_status("loaded CL " + name)
 579         return cl, ''
 580
 581 class LoadCLThread(threading.Thread):
 582         def __init__(self, ui, repo, dir, f, web):
 583                 threading.Thread.__init__(self)
 584                 self.ui = ui
 585                 self.repo = repo
 586                 self.dir = dir
 587                 self.f = f
 588                 self.web = web
 589                 self.cl = None
 590         def run(self):
 591                 cl, err = LoadCL(self.ui, self.repo, self.f[3:], web=self.web)
 592                 if err != '':
 593                         self.ui.warn("loading "+self.dir+self.f+": " + err + "\n")
 594                         return
 595                 self.cl = cl
 596
 597 # Load all the CLs from this repository.
 598 def LoadAllCL(ui, repo, web=True):
 599         dir = CodeReviewDir(ui, repo)
 600         m = {}
 601         files = [f for f in os.listdir(dir) if f.startswith('cl.')]
 602         if not files:
 603                 return m
 604         active = []
 605         first = True
 606         for f in files:
 607                 t = LoadCLThread(ui, repo, dir, f, web)
 608                 t.start()
 609                 if web and first:
 610                         # first request: wait in case it needs to authenticate
 611                         # otherwise we get lots of user/password prompts
 612                         # running in parallel.
 613                         t.join()
 614                         if t.cl:
 615                                 m[t.cl.name] = t.cl
 616                         first = False
 617                 else:
 618                         active.append(t)
 619         for t in active:
 620                 t.join()
 621                 if t.cl:
 622                         m[t.cl.name] = t.cl
 623         return m
 624
 625 # Find repository root.  On error, ui.warn and return None
 626 def RepoDir(ui, repo):
 627         url = repo.url();
 628         if not url.startswith('file:'):
 629                 ui.warn("repository %s is not in local file system\n" % (url,))
 630                 return None
 631         url = url[5:]
 632         if url.endswith('/'):
 633                 url = url[:-1]
 634         typecheck(url, str)
 635         return url
 636
 637 # Find (or make) code review directory.  On error, ui.warn and return None
 638 def CodeReviewDir(ui, repo):
 639         dir = RepoDir(ui, repo)
 640         if dir == None:
 641                 return None
 642         dir += '/.hg/codereview/'
 643         if not os.path.isdir(dir):
 644                 try:
 645                         os.mkdir(dir, 0700)
 646                 except:
 647                         ui.warn('cannot mkdir %s: %s\n' % (dir, ExceptionDetail()))
 648                         return None
 649         typecheck(dir, str)
 650         return dir
 651
 652 # Turn leading tabs into spaces, so that the common white space
 653 # prefix doesn't get confused when people's editors write out
 654 # some lines with spaces, some with tabs.  Only a heuristic
 655 # (some editors don't use 8 spaces either) but a useful one.
 656 def TabsToSpaces(line):
 657         i = 0
 658         while i < len(line) and line[i] == '\t':
 659                 i += 1
 660         return ' '*(8*i) + line[i:]
 661
 662 # Strip maximal common leading white space prefix from text
 663 def StripCommon(text):
 664         typecheck(text, str)
 665         ws = None
 666         for line in text.split('\n'):
 667                 line = line.rstrip()
 668                 if line == '':
 669                         continue
 670                 line = TabsToSpaces(line)
 671                 white = line[:len(line)-len(line.lstrip())]
 672                 if ws == None:
 673                         ws = white
 674                 else:
 675                         common = ''
 676                         for i in range(min(len(white), len(ws))+1):
 677                                 if white[0:i] == ws[0:i]:
 678                                         common = white[0:i]
 679                         ws = common
 680                 if ws == '':
 681                         break
 682         if ws == None:
 683                 return text
 684         t = ''
 685         for line in text.split('\n'):
 686                 line = line.rstrip()
 687                 line = TabsToSpaces(line)
 688                 if line.startswith(ws):
 689                         line = line[len(ws):]
 690                 if line == '' and t == '':
 691                         continue
 692                 t += line + '\n'
 693         while len(t) >= 2 and t[-2:] == '\n\n':
 694                 t = t[:-1]
 695         typecheck(t, str)
 696         return t
 697
 698 # Indent text with indent.
 699 def Indent(text, indent):
 700         typecheck(text, str)
 701         typecheck(indent, str)
 702         t = ''
 703         for line in text.split('\n'):
 704                 t += indent + line + '\n'
 705         typecheck(t, str)
 706         return t
 707
 708 # Return the first line of l
 709 def line1(text):
 710         typecheck(text, str)
 711         return text.split('\n')[0]
 712
 713 _change_prolog = """# Change list.
 714 # Lines beginning with # are ignored.
 715 # Multi-line values should be indented.
 716 """
 717
 718 desc_re = '^(.+: |(tag )?(release|weekly)\.|fix build|undo CL)'
 719
 720 desc_msg = '''Your CL description appears not to use the standard form.
 721
 722 The first line of your change description is conventionally a
 723 one-line summary of the change, prefixed by the primary affected package,
 724 and is used as the subject for code review mail; the rest of the description
 725 elaborates.
 726
 727 Examples:
 728
 729         encoding/rot13: new package
 730
 731         math: add IsInf, IsNaN
 732
 733         net: fix cname in LookupHost
 734
 735         unicode: update to Unicode 5.0.2
 736
 737 '''
 738
 739 def promptyesno(ui, msg):
 740         if hgversion >= "2.7":
 741                 return ui.promptchoice(msg + " $$ &yes $$ &no", 0) == 0
 742         else:
 743                 return ui.promptchoice(msg, ["&yes", "&no"], 0) == 0
 744
 745 def promptremove(ui, repo, f):
 746         if promptyesno(ui, "hg remove %s (y/n)?" % (f,)):
 747                 if hg_commands.remove(ui, repo, 'path:'+f) != 0:
 748                         ui.warn("error removing %s" % (f,))
 749
 750 def promptadd(ui, repo, f):
 751         if promptyesno(ui, "hg add %s (y/n)?" % (f,)):
 752                 if hg_commands.add(ui, repo, 'path:'+f) != 0:
 753                         ui.warn("error adding %s" % (f,))
 754
 755 def EditCL(ui, repo, cl):
 756         set_status(None)        # do not show status
 757         s = cl.EditorText()
 758         while True:
 759                 s = ui.edit(s, ui.username())
 760
 761                 # We can't trust Mercurial + Python not to die before making the change,
 762                 # so, by popular demand, just scribble the most recent CL edit into
 763                 # $(hg root)/last-change so that if Mercurial does die, people
 764                 # can look there for their work.
 765                 try:
 766                         f = open(repo.root+"/last-change", "w")
 767                         f.write(s)
 768                         f.close()
 769                 except:
 770                         pass
 771
 772                 clx, line, err = ParseCL(s, cl.name)
 773                 if err != '':
 774                         if not promptyesno(ui, "error parsing change list: line %d: %s\nre-edit (y/n)?" % (line, err)):
 775                                 return "change list not modified"
 776                         continue
 777
 778                 # Check description.
 779                 if clx.desc == '':
 780                         if promptyesno(ui, "change list should have a description\nre-edit (y/n)?"):
 781                                 continue
 782                 elif re.search('<enter reason for undo>', clx.desc):
 783                         if promptyesno(ui, "change list description omits reason for undo\nre-edit (y/n)?"):
 784                                 continue
 785                 elif not re.match(desc_re, clx.desc.split('\n')[0]):
 786                         if promptyesno(ui, desc_msg + "re-edit (y/n)?"):
 787                                 continue
 788
 789                 # Check file list for files that need to be hg added or hg removed
 790                 # or simply aren't understood.
 791                 pats = ['path:'+f for f in clx.files]
 792                 changed = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
 793                 deleted = hg_matchPattern(ui, repo, *pats, deleted=True)
 794                 unknown = hg_matchPattern(ui, repo, *pats, unknown=True)
 795                 ignored = hg_matchPattern(ui, repo, *pats, ignored=True)
 796                 clean = hg_matchPattern(ui, repo, *pats, clean=True)
 797                 files = []
 798                 for f in clx.files:
 799                         if f in changed:
 800                                 files.append(f)
 801                                 continue
 802                         if f in deleted:
 803                                 promptremove(ui, repo, f)
 804                                 files.append(f)
 805                                 continue
 806                         if f in unknown:
 807                                 promptadd(ui, repo, f)
 808                                 files.append(f)
 809                                 continue
 810                         if f in ignored:
 811                                 ui.warn("error: %s is excluded by .hgignore; omitting\n" % (f,))
 812                                 continue
 813                         if f in clean:
 814                                 ui.warn("warning: %s is listed in the CL but unchanged\n" % (f,))
 815                                 files.append(f)
 816                                 continue
 817                         p = repo.root + '/' + f
 818                         if os.path.isfile(p):
 819                                 ui.warn("warning: %s is a file but not known to hg\n" % (f,))
 820                                 files.append(f)
 821                                 continue
 822                         if os.path.isdir(p):
 823                                 ui.warn("error: %s is a directory, not a file; omitting\n" % (f,))
 824                                 continue
 825                         ui.warn("error: %s does not exist; omitting\n" % (f,))
 826                 clx.files = files
 827
 828                 cl.desc = clx.desc
 829                 cl.reviewer = clx.reviewer
 830                 cl.cc = clx.cc
 831                 cl.files = clx.files
 832                 cl.private = clx.private
 833                 break
 834         return ""
 835
 836 # For use by submit, etc. (NOT by change)
 837 # Get change list number or list of files from command line.
 838 # If files are given, make a new change list.
 839 def CommandLineCL(ui, repo, pats, opts, op="verb", defaultcc=None):
 840         if len(pats) > 0 and GoodCLName(pats[0]):
 841                 if len(pats) != 1:
 842                         return None, "cannot specify change number and file names"
 843                 if opts.get('message'):
 844                         return None, "cannot use -m with existing CL"
 845                 cl, err = LoadCL(ui, repo, pats[0], web=True)
 846                 if err != "":
 847                         return None, err
 848         else:
 849                 cl = CL("new")
 850                 cl.local = True
 851                 cl.files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
 852                 if not cl.files:
 853                         return None, "no files changed (use hg %s <number> to use existing CL)" % op
 854         if opts.get('reviewer'):
 855                 cl.reviewer = Add(cl.reviewer, SplitCommaSpace(opts.get('reviewer')))
 856         if opts.get('cc'):
 857                 cl.cc = Add(cl.cc, SplitCommaSpace(opts.get('cc')))
 858         if defaultcc and not cl.private:
 859                 cl.cc = Add(cl.cc, defaultcc)
 860         if cl.name == "new":
 861                 if opts.get('message'):
 862                         cl.desc = opts.get('message')
 863                 else:
 864                         err = EditCL(ui, repo, cl)
 865                         if err != '':
 866                                 return None, err
 867         return cl, ""
 868
 869 #######################################################################
 870 # Change list file management
 871
 872 # Return list of changed files in repository that match pats.
 873 # The patterns came from the command line, so we warn
 874 # if they have no effect or cannot be understood.
 875 def ChangedFiles(ui, repo, pats, taken=None):
 876         taken = taken or {}
 877         # Run each pattern separately so that we can warn about
 878         # patterns that didn't do anything useful.
 879         for p in pats:
 880                 for f in hg_matchPattern(ui, repo, p, unknown=True):
 881                         promptadd(ui, repo, f)
 882                 for f in hg_matchPattern(ui, repo, p, removed=True):
 883                         promptremove(ui, repo, f)
 884                 files = hg_matchPattern(ui, repo, p, modified=True, added=True, removed=True)
 885                 for f in files:
 886                         if f in taken:
 887                                 ui.warn("warning: %s already in CL %s\n" % (f, taken[f].name))
 888                 if not files:
 889                         ui.warn("warning: %s did not match any modified files\n" % (p,))
 890
 891         # Again, all at once (eliminates duplicates)
 892         l = hg_matchPattern(ui, repo, *pats, modified=True, added=True, removed=True)
 893         l.sort()
 894         if taken:
 895                 l = Sub(l, taken.keys())
 896         return l
 897
 898 # Return list of changed files in repository that match pats and still exist.
 899 def ChangedExistingFiles(ui, repo, pats, opts):
 900         l = hg_matchPattern(ui, repo, *pats, modified=True, added=True)
 901         l.sort()
 902         return l
 903
 904 # Return list of files claimed by existing CLs
 905 def Taken(ui, repo):
 906         all = LoadAllCL(ui, repo, web=False)
 907         taken = {}
 908         for _, cl in all.items():
 909                 for f in cl.files:
 910                         taken[f] = cl
 911         return taken
 912
 913 # Return list of changed files that are not claimed by other CLs
 914 def DefaultFiles(ui, repo, pats):
 915         return ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
 916
 917 #######################################################################
 918 # File format checking.
 919
 920 def CheckFormat(ui, repo, files, just_warn=False):
 921         set_status("running gofmt")
 922         CheckGofmt(ui, repo, files, just_warn)
 923         CheckTabfmt(ui, repo, files, just_warn)
 924
 925 # Check that gofmt run on the list of files does not change them
 926 def CheckGofmt(ui, repo, files, just_warn):
 927         files = gofmt_required(files)
 928         if not files:
 929                 return
 930         cwd = os.getcwd()
 931         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
 932         files = [f for f in files if os.access(f, 0)]
 933         if not files:
 934                 return
 935         try:
 936                 cmd = subprocess.Popen(["gofmt", "-l"] + files, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=sys.platform != "win32")
 937                 cmd.stdin.close()
 938         except:
 939                 raise hg_util.Abort("gofmt: " + ExceptionDetail())
 940         data = cmd.stdout.read()
 941         errors = cmd.stderr.read()
 942         cmd.wait()
 943         set_status("done with gofmt")
 944         if len(errors) > 0:
 945                 ui.warn("gofmt errors:\n" + errors.rstrip() + "\n")
 946                 return
 947         if len(data) > 0:
 948                 msg = "gofmt needs to format these files (run hg gofmt):\n" + Indent(data, "\t").rstrip()
 949                 if just_warn:
 950                         ui.warn("warning: " + msg + "\n")
 951                 else:
 952                         raise hg_util.Abort(msg)
 953         return
 954
 955 # Check that *.[chys] files indent using tabs.
 956 def CheckTabfmt(ui, repo, files, just_warn):
 957         files = [f for f in files if f.startswith('src/') and re.search(r"\.[chys]$", f) and not re.search(r"\.tab\.[ch]$", f)]
 958         if not files:
 959                 return
 960         cwd = os.getcwd()
 961         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
 962         files = [f for f in files if os.access(f, 0)]
 963         badfiles = []
 964         for f in files:
 965                 try:
 966                         for line in open(f, 'r'):
 967                                 # Four leading spaces is enough to complain about,
 968                                 # except that some Plan 9 code uses four spaces as the label indent,
 969                                 # so allow that.
 970                                 if line.startswith('    ') and not re.match('    [A-Za-z0-9_]+:', line):
 971                                         badfiles.append(f)
 972                                         break
 973                 except:
 974                         # ignore cannot open file, etc.
 975                         pass
 976         if len(badfiles) > 0:
 977                 msg = "these files use spaces for indentation (use tabs instead):\n\t" + "\n\t".join(badfiles)
 978                 if just_warn:
 979                         ui.warn("warning: " + msg + "\n")
 980                 else:
 981                         raise hg_util.Abort(msg)
 982         return
 983
 984 #######################################################################
 985 # CONTRIBUTORS file parsing
 986
 987 contributorsCache = None
 988 contributorsURL = None
 989
 990 def ReadContributors(ui, repo):
 991         global contributorsCache
 992         if contributorsCache is not None:
 993                 return contributorsCache
 994
 995         try:
 996                 if contributorsURL is not None:
 997                         opening = contributorsURL
 998                         f = urllib2.urlopen(contributorsURL)
 999                 else:
1000                         opening = repo.root + '/CONTRIBUTORS'
1001                         f = open(repo.root + '/CONTRIBUTORS', 'r')
1002         except:
1003                 ui.write("warning: cannot open %s: %s\n" % (opening, ExceptionDetail()))
1004                 return {}
1005
1006         contributors = {}
1007         for line in f:
1008                 # CONTRIBUTORS is a list of lines like:
1009                 #       Person <email>
1010                 #       Person <email> <alt-email>
1011                 # The first email address is the one used in commit logs.
1012                 if line.startswith('#'):
1013                         continue
1014                 m = re.match(r"([^<>]+\S)\s+(<[^<>\s]+>)((\s+<[^<>\s]+>)*)\s*$", line)
1015                 if m:
1016                         name = m.group(1)
1017                         email = m.group(2)[1:-1]
1018                         contributors[email.lower()] = (name, email)
1019                         for extra in m.group(3).split():
1020                                 contributors[extra[1:-1].lower()] = (name, email)
1021
1022         contributorsCache = contributors
1023         return contributors
1024
1025 def CheckContributor(ui, repo, user=None):
1026         set_status("checking CONTRIBUTORS file")
1027         user, userline = FindContributor(ui, repo, user, warn=False)
1028         if not userline:
1029                 raise hg_util.Abort("cannot find %s in CONTRIBUTORS" % (user,))
1030         return userline
1031
1032 def FindContributor(ui, repo, user=None, warn=True):
1033         if not user:
1034                 user = ui.config("ui", "username")
1035                 if not user:
1036                         raise hg_util.Abort("[ui] username is not configured in .hgrc")
1037         user = user.lower()
1038         m = re.match(r".*<(.*)>", user)
1039         if m:
1040                 user = m.group(1)
1041
1042         contributors = ReadContributors(ui, repo)
1043         if user not in contributors:
1044                 if warn:
1045                         ui.warn("warning: cannot find %s in CONTRIBUTORS\n" % (user,))
1046                 return user, None
1047
1048         user, email = contributors[user]
1049         return email, "%s <%s>" % (user, email)
1050
1051 #######################################################################
1052 # Mercurial helper functions.
1053 # Read http://mercurial.selenic.com/wiki/MercurialApi before writing any of these.
1054 # We use the ui.pushbuffer/ui.popbuffer + hg_commands.xxx tricks for all interaction
1055 # with Mercurial.  It has proved the most stable as they make changes.
1056
1057 hgversion = hg_util.version()
1058
1059 # We require Mercurial 1.9 and suggest Mercurial 2.1.
1060 # The details of the scmutil package changed then,
1061 # so allowing earlier versions would require extra band-aids below.
1062 # Ubuntu 11.10 ships with Mercurial 1.9.1 as the default version.
1063 hg_required = "1.9"
1064 hg_suggested = "2.1"
1065
1066 old_message = """
1067
1068 The code review extension requires Mercurial """+hg_required+""" or newer.
1069 You are using Mercurial """+hgversion+""".
1070
1071 To install a new Mercurial, visit http://mercurial.selenic.com/downloads/.
1072 """
1073
1074 linux_message = """
1075 You may need to clear your current Mercurial installation by running:
1076
1077         sudo apt-get remove mercurial mercurial-common
1078         sudo rm -rf /etc/mercurial
1079 """
1080
1081 if hgversion < hg_required:
1082         msg = old_message
1083         if os.access("/etc/mercurial", 0):
1084                 msg += linux_message
1085         raise hg_util.Abort(msg)
1086
1087 from mercurial.hg import clean as hg_clean
1088 from mercurial import cmdutil as hg_cmdutil
1089 from mercurial import error as hg_error
1090 from mercurial import match as hg_match
1091 from mercurial import node as hg_node
1092
1093 class uiwrap(object):
1094         def __init__(self, ui):
1095                 self.ui = ui
1096                 ui.pushbuffer()
1097                 self.oldQuiet = ui.quiet
1098                 ui.quiet = True
1099                 self.oldVerbose = ui.verbose
1100                 ui.verbose = False
1101         def output(self):
1102                 ui = self.ui
1103                 ui.quiet = self.oldQuiet
1104                 ui.verbose = self.oldVerbose
1105                 return ui.popbuffer()
1106
1107 def to_slash(path):
1108         if sys.platform == "win32":
1109                 return path.replace('\\', '/')
1110         return path
1111
1112 def hg_matchPattern(ui, repo, *pats, **opts):
1113         w = uiwrap(ui)
1114         hg_commands.status(ui, repo, *pats, **opts)
1115         text = w.output()
1116         ret = []
1117         prefix = to_slash(os.path.realpath(repo.root))+'/'
1118         for line in text.split('\n'):
1119                 f = line.split()
1120                 if len(f) > 1:
1121                         if len(pats) > 0:
1122                                 # Given patterns, Mercurial shows relative to cwd
1123                                 p = to_slash(os.path.realpath(f[1]))
1124                                 if not p.startswith(prefix):
1125                                         print >>sys.stderr, "File %s not in repo root %s.\n" % (p, prefix)
1126                                 else:
1127                                         ret.append(p[len(prefix):])
1128                         else:
1129                                 # Without patterns, Mercurial shows relative to root (what we want)
1130                                 ret.append(to_slash(f[1]))
1131         return ret
1132
1133 def hg_heads(ui, repo):
1134         w = uiwrap(ui)
1135         hg_commands.heads(ui, repo)
1136         return w.output()
1137
1138 noise = [
1139         "",
1140         "resolving manifests",
1141         "searching for changes",
1142         "couldn't find merge tool hgmerge",
1143         "adding changesets",
1144         "adding manifests",
1145         "adding file changes",
1146         "all local heads known remotely",
1147 ]
1148
1149 def isNoise(line):
1150         line = str(line)
1151         for x in noise:
1152                 if line == x:
1153                         return True
1154         return False
1155
1156 def hg_incoming(ui, repo):
1157         w = uiwrap(ui)
1158         ret = hg_commands.incoming(ui, repo, force=False, bundle="")
1159         if ret and ret != 1:
1160                 raise hg_util.Abort(ret)
1161         return w.output()
1162
1163 def hg_log(ui, repo, **opts):
1164         for k in ['date', 'keyword', 'rev', 'user']:
1165                 if not opts.has_key(k):
1166                         opts[k] = ""
1167         w = uiwrap(ui)
1168         ret = hg_commands.log(ui, repo, **opts)
1169         if ret:
1170                 raise hg_util.Abort(ret)
1171         return w.output()
1172
1173 def hg_outgoing(ui, repo, **opts):
1174         w = uiwrap(ui)
1175         ret = hg_commands.outgoing(ui, repo, **opts)
1176         if ret and ret != 1:
1177                 raise hg_util.Abort(ret)
1178         return w.output()
1179
1180 def hg_pull(ui, repo, **opts):
1181         w = uiwrap(ui)
1182         ui.quiet = False
1183         ui.verbose = True  # for file list
1184         err = hg_commands.pull(ui, repo, **opts)
1185         for line in w.output().split('\n'):
1186                 if isNoise(line):
1187                         continue
1188                 if line.startswith('moving '):
1189                         line = 'mv ' + line[len('moving '):]
1190                 if line.startswith('getting ') and line.find(' to ') >= 0:
1191                         line = 'mv ' + line[len('getting '):]
1192                 if line.startswith('getting '):
1193                         line = '+ ' + line[len('getting '):]
1194                 if line.startswith('removing '):
1195                         line = '- ' + line[len('removing '):]
1196                 ui.write(line + '\n')
1197         return err
1198
1199 def hg_update(ui, repo, **opts):
1200         w = uiwrap(ui)
1201         ui.quiet = False
1202         ui.verbose = True  # for file list
1203         err = hg_commands.update(ui, repo, **opts)
1204         for line in w.output().split('\n'):
1205                 if isNoise(line):
1206                         continue
1207                 if line.startswith('moving '):
1208                         line = 'mv ' + line[len('moving '):]
1209                 if line.startswith('getting ') and line.find(' to ') >= 0:
1210                         line = 'mv ' + line[len('getting '):]
1211                 if line.startswith('getting '):
1212                         line = '+ ' + line[len('getting '):]
1213                 if line.startswith('removing '):
1214                         line = '- ' + line[len('removing '):]
1215                 ui.write(line + '\n')
1216         return err
1217
1218 def hg_push(ui, repo, **opts):
1219         w = uiwrap(ui)
1220         ui.quiet = False
1221         ui.verbose = True
1222         err = hg_commands.push(ui, repo, **opts)
1223         for line in w.output().split('\n'):
1224                 if not isNoise(line):
1225                         ui.write(line + '\n')
1226         return err
1227
1228 def hg_commit(ui, repo, *pats, **opts):
1229         return hg_commands.commit(ui, repo, *pats, **opts)
1230
1231 #######################################################################
1232 # Mercurial precommit hook to disable commit except through this interface.
1233
1234 commit_okay = False
1235
1236 def precommithook(ui, repo, **opts):
1237         if hgversion >= "2.1":
1238                 from mercurial import phases
1239                 if repo.ui.config('phases', 'new-commit') >= phases.secret:
1240                         return False
1241         if commit_okay:
1242                 return False  # False means okay.
1243         ui.write("\ncodereview extension enabled; use mail, upload, or submit instead of commit\n\n")
1244         return True
1245
1246 #######################################################################
1247 # @clnumber file pattern support
1248
1249 # We replace scmutil.match with the MatchAt wrapper to add the @clnumber pattern.
1250
1251 match_repo = None
1252 match_ui = None
1253 match_orig = None
1254
1255 def InstallMatch(ui, repo):
1256         global match_repo
1257         global match_ui
1258         global match_orig
1259
1260         match_ui = ui
1261         match_repo = repo
1262
1263         from mercurial import scmutil
1264         match_orig = scmutil.match
1265         scmutil.match = MatchAt
1266
1267 def MatchAt(ctx, pats=None, opts=None, globbed=False, default='relpath'):
1268         taken = []
1269         files = []
1270         pats = pats or []
1271         opts = opts or {}
1272
1273         for p in pats:
1274                 if p.startswith('@'):
1275                         taken.append(p)
1276                         clname = p[1:]
1277                         if clname == "default":
1278                                 files = DefaultFiles(match_ui, match_repo, [])
1279                         else:
1280                                 if not GoodCLName(clname):
1281                                         raise hg_util.Abort("invalid CL name " + clname)
1282                                 cl, err = LoadCL(match_repo.ui, match_repo, clname, web=False)
1283                                 if err != '':
1284                                         raise hg_util.Abort("loading CL " + clname + ": " + err)
1285                                 if not cl.files:
1286                                         raise hg_util.Abort("no files in CL " + clname)
1287                                 files = Add(files, cl.files)
1288         pats = Sub(pats, taken) + ['path:'+f for f in files]
1289
1290         # work-around for http://selenic.com/hg/rev/785bbc8634f8
1291         if not hasattr(ctx, 'match'):
1292                 ctx = ctx[None]
1293         return match_orig(ctx, pats=pats, opts=opts, globbed=globbed, default=default)
1294
1295 #######################################################################
1296 # Commands added by code review extension.
1297
1298 def hgcommand(f):
1299         return f
1300
1301 #######################################################################
1302 # hg change
1303
1304 @hgcommand
1305 def change(ui, repo, *pats, **opts):
1306         """create, edit or delete a change list
1307
1308         Create, edit or delete a change list.
1309         A change list is a group of files to be reviewed and submitted together,
1310         plus a textual description of the change.
1311         Change lists are referred to by simple alphanumeric names.
1312
1313         Changes must be reviewed before they can be submitted.
1314
1315         In the absence of options, the change command opens the
1316         change list for editing in the default editor.
1317
1318         Deleting a change with the -d or -D flag does not affect
1319         the contents of the files listed in that change.  To revert
1320         the files listed in a change, use
1321
1322                 hg revert @123456
1323
1324         before running hg change -d 123456.
1325         """
1326
1327         if codereview_disabled:
1328                 raise hg_util.Abort(codereview_disabled)
1329
1330         dirty = {}
1331         if len(pats) > 0 and GoodCLName(pats[0]):
1332                 name = pats[0]
1333                 if len(pats) != 1:
1334                         raise hg_util.Abort("cannot specify CL name and file patterns")
1335                 pats = pats[1:]
1336                 cl, err = LoadCL(ui, repo, name, web=True)
1337                 if err != '':
1338                         raise hg_util.Abort(err)
1339                 if not cl.local and (opts["stdin"] or not opts["stdout"]):
1340                         raise hg_util.Abort("cannot change non-local CL " + name)
1341         else:
1342                 name = "new"
1343                 cl = CL("new")
1344                 if not workbranch(repo[None].branch()):
1345                         raise hg_util.Abort("cannot create CL outside default branch; switch with 'hg update default'")
1346                 dirty[cl] = True
1347                 files = ChangedFiles(ui, repo, pats, taken=Taken(ui, repo))
1348
1349         if opts["delete"] or opts["deletelocal"]:
1350                 if opts["delete"] and opts["deletelocal"]:
1351                         raise hg_util.Abort("cannot use -d and -D together")
1352                 flag = "-d"
1353                 if opts["deletelocal"]:
1354                         flag = "-D"
1355                 if name == "new":
1356                         raise hg_util.Abort("cannot use "+flag+" with file patterns")
1357                 if opts["stdin"] or opts["stdout"]:
1358                         raise hg_util.Abort("cannot use "+flag+" with -i or -o")
1359                 if not cl.local:
1360                         raise hg_util.Abort("cannot change non-local CL " + name)
1361                 if opts["delete"]:
1362                         if cl.copied_from:
1363                                 raise hg_util.Abort("original author must delete CL; hg change -D will remove locally")
1364                         PostMessage(ui, cl.name, "*** Abandoned ***", send_mail=cl.mailed)
1365                         EditDesc(cl.name, closed=True, private=cl.private)
1366                 cl.Delete(ui, repo)
1367                 return
1368
1369         if opts["stdin"]:
1370                 s = sys.stdin.read()
1371                 clx, line, err = ParseCL(s, name)
1372                 if err != '':
1373                         raise hg_util.Abort("error parsing change list: line %d: %s" % (line, err))
1374                 if clx.desc is not None:
1375                         cl.desc = clx.desc;
1376                         dirty[cl] = True
1377                 if clx.reviewer is not None:
1378                         cl.reviewer = clx.reviewer
1379                         dirty[cl] = True
1380                 if clx.cc is not None:
1381                         cl.cc = clx.cc
1382                         dirty[cl] = True
1383                 if clx.files is not None:
1384                         cl.files = clx.files
1385                         dirty[cl] = True
1386                 if clx.private != cl.private:
1387                         cl.private = clx.private
1388                         dirty[cl] = True
1389
1390         if not opts["stdin"] and not opts["stdout"]:
1391                 if name == "new":
1392                         cl.files = files
1393                 err = EditCL(ui, repo, cl)
1394                 if err != "":
1395                         raise hg_util.Abort(err)
1396                 dirty[cl] = True
1397
1398         for d, _ in dirty.items():
1399                 name = d.name
1400                 d.Flush(ui, repo)
1401                 if name == "new":
1402                         d.Upload(ui, repo, quiet=True)
1403
1404         if opts["stdout"]:
1405                 ui.write(cl.EditorText())
1406         elif opts["pending"]:
1407                 ui.write(cl.PendingText())
1408         elif name == "new":
1409                 if ui.quiet:
1410                         ui.write(cl.name)
1411                 else:
1412                         ui.write("CL created: " + cl.url + "\n")
1413         return
1414
1415 #######################################################################
1416 # hg code-login (broken?)
1417
1418 @hgcommand
1419 def code_login(ui, repo, **opts):
1420         """log in to code review server
1421
1422         Logs in to the code review server, saving a cookie in
1423         a file in your home directory.
1424         """
1425         if codereview_disabled:
1426                 raise hg_util.Abort(codereview_disabled)
1427
1428         MySend(None)
1429
1430 #######################################################################
1431 # hg clpatch / undo / release-apply / download
1432 # All concerned with applying or unapplying patches to the repository.
1433
1434 @hgcommand
1435 def clpatch(ui, repo, clname, **opts):
1436         """import a patch from the code review server
1437
1438         Imports a patch from the code review server into the local client.
1439         If the local client has already modified any of the files that the
1440         patch modifies, this command will refuse to apply the patch.
1441
1442         Submitting an imported patch will keep the original author's
1443         name as the Author: line but add your own name to a Committer: line.
1444         """
1445         if not workbranch(repo[None].branch()):
1446                 raise hg_util.Abort("cannot run hg clpatch outside default branch")
1447         err = clpatch_or_undo(ui, repo, clname, opts, mode="clpatch")
1448         if err:
1449                 raise hg_util.Abort(err)
1450
1451 @hgcommand
1452 def undo(ui, repo, clname, **opts):
1453         """undo the effect of a CL
1454
1455         Creates a new CL that undoes an earlier CL.
1456         After creating the CL, opens the CL text for editing so that
1457         you can add the reason for the undo to the description.
1458         """
1459         if not workbranch(repo[None].branch()):
1460                 raise hg_util.Abort("cannot run hg undo outside default branch")
1461         err = clpatch_or_undo(ui, repo, clname, opts, mode="undo")
1462         if err:
1463                 raise hg_util.Abort(err)
1464
1465 @hgcommand
1466 def release_apply(ui, repo, clname, **opts):
1467         """apply a CL to the release branch
1468
1469         Creates a new CL copying a previously committed change
1470         from the main branch to the release branch.
1471         The current client must either be clean or already be in
1472         the release branch.
1473
1474         The release branch must be created by starting with a
1475         clean client, disabling the code review plugin, and running:
1476
1477                 hg update weekly.YYYY-MM-DD
1478                 hg branch release-branch.rNN
1479                 hg commit -m 'create release-branch.rNN'
1480                 hg push --new-branch
1481
1482         Then re-enable the code review plugin.
1483
1484         People can test the release branch by running
1485
1486                 hg update release-branch.rNN
1487
1488         in a clean client.  To return to the normal tree,
1489
1490                 hg update default
1491
1492         Move changes since the weekly into the release branch
1493         using hg release-apply followed by the usual code review
1494         process and hg submit.
1495
1496         When it comes time to tag the release, record the
1497         final long-form tag of the release-branch.rNN
1498         in the *default* branch's .hgtags file.  That is, run
1499
1500                 hg update default
1501
1502         and then edit .hgtags as you would for a weekly.
1503
1504         """
1505         c = repo[None]
1506         if not releaseBranch:
1507                 raise hg_util.Abort("no active release branches")
1508         if c.branch() != releaseBranch:
1509                 if c.modified() or c.added() or c.removed():
1510                         raise hg_util.Abort("uncommitted local changes - cannot switch branches")
1511                 err = hg_clean(repo, releaseBranch)
1512                 if err:
1513                         raise hg_util.Abort(err)
1514         try:
1515                 err = clpatch_or_undo(ui, repo, clname, opts, mode="backport")
1516                 if err:
1517                         raise hg_util.Abort(err)
1518         except Exception, e:
1519                 hg_clean(repo, "default")
1520                 raise e
1521
1522 def rev2clname(rev):
1523         # Extract CL name from revision description.
1524         # The last line in the description that is a codereview URL is the real one.
1525         # Earlier lines might be part of the user-written description.
1526         all = re.findall('(?m)^https?://codereview.appspot.com/([0-9]+)$', rev.description())
1527         if len(all) > 0:
1528                 return all[-1]
1529         return ""
1530
1531 undoHeader = """undo CL %s / %s
1532
1533 <enter reason for undo>
1534
1535 ««« original CL description
1536 """
1537
1538 undoFooter = """
1539 »»»
1540 """
1541
1542 backportHeader = """[%s] %s
1543
1544 ««« CL %s / %s
1545 """
1546
1547 backportFooter = """
1548 »»»
1549 """
1550
1551 # Implementation of clpatch/undo.
1552 def clpatch_or_undo(ui, repo, clname, opts, mode):
1553         if codereview_disabled:
1554                 return codereview_disabled
1555
1556         if mode == "undo" or mode == "backport":
1557                 # Find revision in Mercurial repository.
1558                 # Assume CL number is 7+ decimal digits.
1559                 # Otherwise is either change log sequence number (fewer decimal digits),
1560                 # hexadecimal hash, or tag name.
1561                 # Mercurial will fall over long before the change log
1562                 # sequence numbers get to be 7 digits long.
1563                 if re.match('^[0-9]{7,}$', clname):
1564                         found = False
1565                         for r in hg_log(ui, repo, keyword="codereview.appspot.com/"+clname, limit=100, template="{node}\n").split():
1566                                 rev = repo[r]
1567                                 # Last line with a code review URL is the actual review URL.
1568                                 # Earlier ones might be part of the CL description.
1569                                 n = rev2clname(rev)
1570                                 if n == clname:
1571                                         found = True
1572                                         break
1573                         if not found:
1574                                 return "cannot find CL %s in local repository" % clname
1575                 else:
1576                         rev = repo[clname]
1577                         if not rev:
1578                                 return "unknown revision %s" % clname
1579                         clname = rev2clname(rev)
1580                         if clname == "":
1581                                 return "cannot find CL name in revision description"
1582
1583                 # Create fresh CL and start with patch that would reverse the change.
1584                 vers = hg_node.short(rev.node())
1585                 cl = CL("new")
1586                 desc = str(rev.description())
1587                 if mode == "undo":
1588                         cl.desc = (undoHeader % (clname, vers)) + desc + undoFooter
1589                 else:
1590                         cl.desc = (backportHeader % (releaseBranch, line1(desc), clname, vers)) + desc + undoFooter
1591                 v1 = vers
1592                 v0 = hg_node.short(rev.parents()[0].node())
1593                 if mode == "undo":
1594                         arg = v1 + ":" + v0
1595                 else:
1596                         vers = v0
1597                         arg = v0 + ":" + v1
1598                 patch = RunShell(["hg", "diff", "--git", "-r", arg])
1599
1600         else:  # clpatch
1601                 cl, vers, patch, err = DownloadCL(ui, repo, clname)
1602                 if err != "":
1603                         return err
1604                 if patch == emptydiff:
1605                         return "codereview issue %s has no diff" % clname
1606
1607         # find current hg version (hg identify)
1608         ctx = repo[None]
1609         parents = ctx.parents()
1610         id = '+'.join([hg_node.short(p.node()) for p in parents])
1611
1612         # if version does not match the patch version,
1613         # try to update the patch line numbers.
1614         if vers != "" and id != vers:
1615                 # "vers in repo" gives the wrong answer
1616                 # on some versions of Mercurial.  Instead, do the actual
1617                 # lookup and catch the exception.
1618                 try:
1619                         repo[vers].description()
1620                 except:
1621                         return "local repository is out of date; sync to get %s" % (vers)
1622                 patch1, err = portPatch(repo, patch, vers, id)
1623                 if err != "":
1624                         if not opts["ignore_hgapplydiff_failure"]:
1625                                 return "codereview issue %s is out of date: %s (%s->%s)" % (clname, err, vers, id)
1626                 else:
1627                         patch = patch1
1628         argv = ["hgapplydiff"]
1629         if opts["no_incoming"] or mode == "backport":
1630                 argv += ["--checksync=false"]
1631         try:
1632                 cmd = subprocess.Popen(argv, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None, close_fds=sys.platform != "win32")
1633         except:
1634                 return "hgapplydiff: " + ExceptionDetail() + "\nInstall hgapplydiff with:\n$ go get golang.org/x/codereview/cmd/hgapplydiff\n"
1635
1636         out, err = cmd.communicate(patch)
1637         if cmd.returncode != 0 and not opts["ignore_hgapplydiff_failure"]:
1638                 return "hgapplydiff failed"
1639         cl.local = True
1640         cl.files = out.strip().split()
1641         if not cl.files and not opts["ignore_hgapplydiff_failure"]:
1642                 return "codereview issue %s has no changed files" % clname
1643         files = ChangedFiles(ui, repo, [])
1644         extra = Sub(cl.files, files)
1645         if extra:
1646                 ui.warn("warning: these files were listed in the patch but not changed:\n\t" + "\n\t".join(extra) + "\n")
1647         cl.Flush(ui, repo)
1648         if mode == "undo":
1649                 err = EditCL(ui, repo, cl)
1650                 if err != "":
1651                         return "CL created, but error editing: " + err
1652                 cl.Flush(ui, repo)
1653         else:
1654                 ui.write(cl.PendingText() + "\n")
1655
1656 # portPatch rewrites patch from being a patch against
1657 # oldver to being a patch against newver.
1658 def portPatch(repo, patch, oldver, newver):
1659         lines = patch.splitlines(True) # True = keep \n
1660         delta = None
1661         for i in range(len(lines)):
1662                 line = lines[i]
1663                 if line.startswith('--- a/'):
1664                         file = line[6:-1]
1665                         delta = fileDeltas(repo, file, oldver, newver)
1666                 if not delta or not line.startswith('@@ '):
1667                         continue
1668                 # @@ -x,y +z,w @@ means the patch chunk replaces
1669                 # the original file's line numbers x up to x+y with the
1670                 # line numbers z up to z+w in the new file.
1671                 # Find the delta from x in the original to the same
1672                 # line in the current version and add that delta to both
1673                 # x and z.
1674                 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1675                 if not m:
1676                         return None, "error parsing patch line numbers"
1677                 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1678                 d, err = lineDelta(delta, n1, len1)
1679                 if err != "":
1680                         return "", err
1681                 n1 += d
1682                 n2 += d
1683                 lines[i] = "@@ -%d,%d +%d,%d @@\n" % (n1, len1, n2, len2)
1684
1685         newpatch = ''.join(lines)
1686         return newpatch, ""
1687
1688 # fileDelta returns the line number deltas for the given file's
1689 # changes from oldver to newver.
1690 # The deltas are a list of (n, len, newdelta) triples that say
1691 # lines [n, n+len) were modified, and after that range the
1692 # line numbers are +newdelta from what they were before.
1693 def fileDeltas(repo, file, oldver, newver):
1694         cmd = ["hg", "diff", "--git", "-r", oldver + ":" + newver, "path:" + file]
1695         data = RunShell(cmd, silent_ok=True)
1696         deltas = []
1697         for line in data.splitlines():
1698                 m = re.match('@@ -([0-9]+),([0-9]+) \+([0-9]+),([0-9]+) @@', line)
1699                 if not m:
1700                         continue
1701                 n1, len1, n2, len2 = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4))
1702                 deltas.append((n1, len1, n2+len2-(n1+len1)))
1703         return deltas
1704
1705 # lineDelta finds the appropriate line number delta to apply to the lines [n, n+len).
1706 # It returns an error if those lines were rewritten by the patch.
1707 def lineDelta(deltas, n, len):
1708         d = 0
1709         for (old, oldlen, newdelta) in deltas:
1710                 if old >= n+len:
1711                         break
1712                 if old+len > n:
1713                         return 0, "patch and recent changes conflict"
1714                 d = newdelta
1715         return d, ""
1716
1717 @hgcommand
1718 def download(ui, repo, clname, **opts):
1719         """download a change from the code review server
1720
1721         Download prints a description of the given change list
1722         followed by its diff, downloaded from the code review server.
1723         """
1724         if codereview_disabled:
1725                 raise hg_util.Abort(codereview_disabled)
1726
1727         cl, vers, patch, err = DownloadCL(ui, repo, clname)
1728         if err != "":
1729                 return err
1730         ui.write(cl.EditorText() + "\n")
1731         ui.write(patch + "\n")
1732         return
1733
1734 #######################################################################
1735 # hg file
1736
1737 @hgcommand
1738 def file(ui, repo, clname, pat, *pats, **opts):
1739         """assign files to or remove files from a change list
1740
1741         Assign files to or (with -d) remove files from a change list.
1742
1743         The -d option only removes files from the change list.
1744         It does not edit them or remove them from the repository.
1745         """
1746         if codereview_disabled:
1747                 raise hg_util.Abort(codereview_disabled)
1748
1749         pats = tuple([pat] + list(pats))
1750         if not GoodCLName(clname):
1751                 return "invalid CL name " + clname
1752
1753         dirty = {}
1754         cl, err = LoadCL(ui, repo, clname, web=False)
1755         if err != '':
1756                 return err
1757         if not cl.local:
1758                 return "cannot change non-local CL " + clname
1759
1760         files = ChangedFiles(ui, repo, pats)
1761
1762         if opts["delete"]:
1763                 oldfiles = Intersect(files, cl.files)
1764                 if oldfiles:
1765                         if not ui.quiet:
1766                                 ui.status("# Removing files from CL.  To undo:\n")
1767                                 ui.status("#    cd %s\n" % (repo.root))
1768                                 for f in oldfiles:
1769                                         ui.status("#    hg file %s %s\n" % (cl.name, f))
1770                         cl.files = Sub(cl.files, oldfiles)
1771                         cl.Flush(ui, repo)
1772                 else:
1773                         ui.status("no such files in CL")
1774                 return
1775
1776         if not files:
1777                 return "no such modified files"
1778
1779         files = Sub(files, cl.files)
1780         taken = Taken(ui, repo)
1781         warned = False
1782         for f in files:
1783                 if f in taken:
1784                         if not warned and not ui.quiet:
1785                                 ui.status("# Taking files from other CLs.  To undo:\n")
1786                                 ui.status("#    cd %s\n" % (repo.root))
1787                                 warned = True
1788                         ocl = taken[f]
1789                         if not ui.quiet:
1790                                 ui.status("#    hg file %s %s\n" % (ocl.name, f))
1791                         if ocl not in dirty:
1792                                 ocl.files = Sub(ocl.files, files)
1793                                 dirty[ocl] = True
1794         cl.files = Add(cl.files, files)
1795         dirty[cl] = True
1796         for d, _ in dirty.items():
1797                 d.Flush(ui, repo)
1798         return
1799
1800 #######################################################################
1801 # hg gofmt
1802
1803 @hgcommand
1804 def gofmt(ui, repo, *pats, **opts):
1805         """apply gofmt to modified files
1806
1807         Applies gofmt to the modified files in the repository that match
1808         the given patterns.
1809         """
1810         if codereview_disabled:
1811                 raise hg_util.Abort(codereview_disabled)
1812
1813         files = ChangedExistingFiles(ui, repo, pats, opts)
1814         files = gofmt_required(files)
1815         if not files:
1816                 ui.status("no modified go files\n")
1817                 return
1818         cwd = os.getcwd()
1819         files = [RelativePath(repo.root + '/' + f, cwd) for f in files]
1820         try:
1821                 cmd = ["gofmt", "-l"]
1822                 if not opts["list"]:
1823                         cmd += ["-w"]
1824                 if subprocess.call(cmd + files) != 0:
1825                         raise hg_util.Abort("gofmt did not exit cleanly")
1826         except hg_error.Abort, e:
1827                 raise
1828         except:
1829                 raise hg_util.Abort("gofmt: " + ExceptionDetail())
1830         return
1831
1832 def gofmt_required(files):
1833         return [f for f in files if (not f.startswith('test/') or f.startswith('test/bench/')) and f.endswith('.go')]
1834
1835 #######################################################################
1836 # hg mail
1837
1838 @hgcommand
1839 def mail(ui, repo, *pats, **opts):
1840         """mail a change for review
1841
1842         Uploads a patch to the code review server and then sends mail
1843         to the reviewer and CC list asking for a review.
1844         """
1845         if codereview_disabled:
1846                 raise hg_util.Abort(codereview_disabled)
1847
1848         cl, err = CommandLineCL(ui, repo, pats, opts, op="mail", defaultcc=defaultcc)
1849         if err != "":
1850                 raise hg_util.Abort(err)
1851         cl.Upload(ui, repo, gofmt_just_warn=True)
1852         if not cl.reviewer:
1853                 # If no reviewer is listed, assign the review to defaultcc.
1854                 # This makes sure that it appears in the
1855                 # codereview.appspot.com/user/defaultcc
1856                 # page, so that it doesn't get dropped on the floor.
1857                 if not defaultcc or cl.private:
1858                         raise hg_util.Abort("no reviewers listed in CL")
1859                 cl.cc = Sub(cl.cc, defaultcc)
1860                 cl.reviewer = defaultcc
1861                 cl.Flush(ui, repo)
1862
1863         if cl.files == []:
1864                         raise hg_util.Abort("no changed files, not sending mail")
1865
1866         cl.Mail(ui, repo)
1867
1868 #######################################################################
1869 # hg p / hg pq / hg ps / hg pending
1870
1871 @hgcommand
1872 def ps(ui, repo, *pats, **opts):
1873         """alias for hg p --short
1874         """
1875         opts['short'] = True
1876         return pending(ui, repo, *pats, **opts)
1877
1878 @hgcommand
1879 def pq(ui, repo, *pats, **opts):
1880         """alias for hg p --quick
1881         """
1882         opts['quick'] = True
1883         return pending(ui, repo, *pats, **opts)
1884
1885 @hgcommand
1886 def pending(ui, repo, *pats, **opts):
1887         """show pending changes
1888
1889         Lists pending changes followed by a list of unassigned but modified files.
1890         """
1891         if codereview_disabled:
1892                 raise hg_util.Abort(codereview_disabled)
1893
1894         quick = opts.get('quick', False)
1895         short = opts.get('short', False)
1896         m = LoadAllCL(ui, repo, web=not quick and not short)
1897         names = m.keys()
1898         names.sort()
1899         for name in names:
1900                 cl = m[name]
1901                 if short:
1902                         ui.write(name + "\t" + line1(cl.desc) + "\n")
1903                 else:
1904                         ui.write(cl.PendingText(quick=quick) + "\n")
1905
1906         if short:
1907                 return 0
1908         files = DefaultFiles(ui, repo, [])
1909         if len(files) > 0:
1910                 s = "Changed files not in any CL:\n"
1911                 for f in files:
1912                         s += "\t" + f + "\n"
1913                 ui.write(s)
1914
1915 #######################################################################
1916 # hg submit
1917
1918 def need_sync():
1919         raise hg_util.Abort("local repository out of date; must sync before submit")
1920
1921 def branch_prefix(ui, repo):
1922         prefix = ""
1923         branch = repo[None].branch()
1924         if branch.startswith("dev."):
1925                 prefix = "[" + branch + "] "
1926         return prefix
1927
1928 @hgcommand
1929 def submit(ui, repo, *pats, **opts):
1930         """submit change to remote repository
1931
1932         Submits change to remote repository.
1933         Bails out if the local repository is not in sync with the remote one.
1934         """
1935         if codereview_disabled:
1936                 raise hg_util.Abort(codereview_disabled)
1937
1938         # We already called this on startup but sometimes Mercurial forgets.
1939         set_mercurial_encoding_to_utf8()
1940
1941         if not opts["no_incoming"] and hg_incoming(ui, repo):
1942                 need_sync()
1943
1944         cl, err = CommandLineCL(ui, repo, pats, opts, op="submit", defaultcc=defaultcc)
1945         if err != "":
1946                 raise hg_util.Abort(err)
1947
1948         user = None
1949         if cl.copied_from:
1950                 user = cl.copied_from
1951         userline = CheckContributor(ui, repo, user)
1952         typecheck(userline, str)
1953
1954         about = ""
1955
1956         if not cl.lgtm and not opts.get('tbr') and needLGTM(cl):
1957                 raise hg_util.Abort("this CL has not been LGTM'ed")
1958         if cl.lgtm:
1959                 about += "LGTM=" + JoinComma([CutDomain(who) for (who, line, approval) in cl.lgtm if approval]) + "\n"
1960         reviewer = cl.reviewer
1961         if opts.get('tbr'):
1962                 tbr = SplitCommaSpace(opts.get('tbr'))
1963                 for name in tbr:
1964                         if name.startswith('golang-'):
1965                                 raise hg_util.Abort("--tbr requires a person, not a mailing list")
1966                 cl.reviewer = Add(cl.reviewer, tbr)
1967                 about += "TBR=" + JoinComma([CutDomain(s) for s in tbr]) + "\n"
1968         if reviewer:
1969                 about += "R=" + JoinComma([CutDomain(s) for s in reviewer]) + "\n"
1970         if cl.cc:
1971                 about += "CC=" + JoinComma([CutDomain(s) for s in cl.cc]) + "\n"
1972
1973         if not cl.reviewer and needLGTM(cl):
1974                 raise hg_util.Abort("no reviewers listed in CL")
1975
1976         if not cl.local:
1977                 raise hg_util.Abort("cannot submit non-local CL")
1978
1979         # upload, to sync current patch and also get change number if CL is new.
1980         if not cl.copied_from:
1981                 cl.Upload(ui, repo, gofmt_just_warn=True)
1982
1983         # check gofmt for real; allowed upload to warn in order to save CL.
1984         cl.Flush(ui, repo)
1985         CheckFormat(ui, repo, cl.files)
1986
1987         about += "%s%s\n" % (server_url_base, cl.name)
1988
1989         if cl.copied_from:
1990                 about += "\nCommitter: " + CheckContributor(ui, repo, None) + "\n"
1991         typecheck(about, str)
1992
1993         if not cl.mailed and not cl.copied_from:                # in case this is TBR
1994                 cl.Mail(ui, repo)
1995
1996         # submit changes locally
1997         message = branch_prefix(ui, repo) + cl.desc.rstrip() + "\n\n" + about
1998         typecheck(message, str)
1999
2000         set_status("pushing " + cl.name + " to remote server")
2001
2002         if hg_outgoing(ui, repo):
2003                 raise hg_util.Abort("local repository corrupt or out-of-phase with remote: found outgoing changes")
2004
2005         old_heads = len(hg_heads(ui, repo).split())
2006
2007         # Normally we commit listing the specific files in the CL.
2008         # If there are no changed files other than those in the CL, however,
2009         # let hg build the list, because then committing a merge works.
2010         # (You cannot name files for a merge commit, even if you name
2011         # all the files that would be committed by not naming any.)
2012         files = ['path:'+f for f in cl.files]
2013         if ChangedFiles(ui, repo, []) == cl.files:
2014                 files = []
2015
2016         global commit_okay
2017         commit_okay = True
2018         ret = hg_commit(ui, repo, *files, message=message, user=userline)
2019         commit_okay = False
2020         if ret:
2021                 raise hg_util.Abort("nothing changed")
2022
2023         node = repo["-1"].node()
2024         # push to remote; if it fails for any reason, roll back
2025         try:
2026                 new_heads = len(hg_heads(ui, repo).split())
2027                 if cl.desc.find("create new branch") < 0 and old_heads != new_heads and not (old_heads == 0 and new_heads == 1):
2028                         # Created new head, so we weren't up to date.
2029                         need_sync()
2030
2031                 # Push changes to remote.  If it works, we're committed.  If not, roll back.
2032                 try:
2033                         if hg_push(ui, repo, new_branch=cl.desc.find("create new branch")>=0):
2034                                 raise hg_util.Abort("push error")
2035                 except hg_error.Abort, e:
2036                         if e.message.find("push creates new heads") >= 0:
2037                                 # Remote repository had changes we missed.
2038                                 need_sync()
2039                         raise
2040                 except urllib2.HTTPError, e:
2041                         print >>sys.stderr, "pushing to remote server failed; do you have commit permissions?"
2042                         raise
2043         except:
2044                 real_rollback()
2045                 raise
2046
2047         # We're committed. Upload final patch, close review, add commit message.
2048         changeURL = hg_node.short(node)
2049         url = ui.expandpath("default")
2050         m = re.match("(^https?://([^@/]+@)?([^.]+)\.googlecode\.com/hg/?)" + "|" +
2051                 "(^https?://([^@/]+@)?code\.google\.com/p/([^/.]+)(\.[^./]+)?/?)", url)
2052         if m:
2053                 if m.group(1): # prj.googlecode.com/hg/ case
2054                         changeURL = "https://code.google.com/p/%s/source/detail?r=%s" % (m.group(3), changeURL)
2055                 elif m.group(4) and m.group(7): # code.google.com/p/prj.subrepo/ case
2056                         changeURL = "https://code.google.com/p/%s/source/detail?r=%s&repo=%s" % (m.group(6), changeURL, m.group(7)[1:])
2057                 elif m.group(4): # code.google.com/p/prj/ case
2058                         changeURL = "https://code.google.com/p/%s/source/detail?r=%s" % (m.group(6), changeURL)
2059                 else:
2060                         print >>sys.stderr, "URL: ", url
2061         else:
2062                 print >>sys.stderr, "URL: ", url
2063         pmsg = "*** Submitted as " + changeURL + " ***\n\n" + message
2064
2065         # When posting, move reviewers to CC line,
2066         # so that the issue stops showing up in their "My Issues" page.
2067         PostMessage(ui, cl.name, pmsg, reviewers="", cc=JoinComma(cl.reviewer+cl.cc))
2068
2069         if not cl.copied_from:
2070                 EditDesc(cl.name, closed=True, private=cl.private)
2071         cl.Delete(ui, repo)
2072
2073         c = repo[None]
2074         if c.branch() == releaseBranch and not c.modified() and not c.added() and not c.removed():
2075                 ui.write("switching from %s to default branch.\n" % releaseBranch)
2076                 err = hg_clean(repo, "default")
2077                 if err:
2078                         return err
2079         return 0
2080
2081 def needLGTM(cl):
2082         rev = cl.reviewer
2083         isGobot = 'gobot' in rev or 'gobot@swtch.com' in rev or 'gobot@golang.org' in rev
2084
2085         # A+C CLs generated by addca do not need LGTM
2086         if cl.desc.startswith('A+C:') and 'Generated by a+c.' in cl.desc and isGobot:
2087                 return False
2088
2089         # CLs modifying only go1.x.txt do not need LGTM
2090         if len(cl.files) == 1 and cl.files[0].startswith('doc/go1.') and cl.files[0].endswith('.txt'):
2091                 return False
2092
2093         # Other CLs need LGTM
2094         return True
2095
2096 #######################################################################
2097 # hg sync
2098
2099 @hgcommand
2100 def sync(ui, repo, **opts):
2101         """synchronize with remote repository
2102
2103         Incorporates recent changes from the remote repository
2104         into the local repository.
2105         """
2106         if codereview_disabled:
2107                 raise hg_util.Abort(codereview_disabled)
2108
2109         if not opts["local"]:
2110                 # If there are incoming CLs, pull -u will do the update.
2111                 # If there are no incoming CLs, do hg update to make sure
2112                 # that an update always happens regardless. This is less
2113                 # surprising than update depending on incoming CLs.
2114                 # It is important not to do both hg pull -u and hg update
2115                 # in the same command, because the hg update will end
2116                 # up marking resolve conflicts from the hg pull -u as resolved,
2117                 # causing files with <<< >>> markers to not show up in
2118                 # hg resolve -l. Yay Mercurial.
2119                 if hg_incoming(ui, repo):
2120                         err = hg_pull(ui, repo, update=True)
2121                 else:
2122                         err = hg_update(ui, repo)
2123                 if err:
2124                         return err
2125         sync_changes(ui, repo)
2126
2127 def sync_changes(ui, repo):
2128         # Look through recent change log descriptions to find
2129         # potential references to http://.*/our-CL-number.
2130         # Double-check them by looking at the Rietveld log.
2131         for rev in hg_log(ui, repo, limit=100, template="{node}\n").split():
2132                 desc = repo[rev].description().strip()
2133                 for clname in re.findall('(?m)^https?://(?:[^\n]+)/([0-9]+)$', desc):
2134                         if IsLocalCL(ui, repo, clname) and IsRietveldSubmitted(ui, clname, repo[rev].hex()):
2135                                 ui.warn("CL %s submitted as %s; closing\n" % (clname, repo[rev]))
2136                                 cl, err = LoadCL(ui, repo, clname, web=False)
2137                                 if err != "":
2138                                         ui.warn("loading CL %s: %s\n" % (clname, err))
2139                                         continue
2140                                 if not cl.copied_from:
2141                                         EditDesc(cl.name, closed=True, private=cl.private)
2142                                 cl.Delete(ui, repo)
2143
2144         # Remove files that are not modified from the CLs in which they appear.
2145         all = LoadAllCL(ui, repo, web=False)
2146         changed = ChangedFiles(ui, repo, [])
2147         for cl in all.values():
2148                 extra = Sub(cl.files, changed)
2149                 if extra:
2150                         ui.warn("Removing unmodified files from CL %s:\n" % (cl.name,))
2151                         for f in extra:
2152                                 ui.warn("\t%s\n" % (f,))
2153                         cl.files = Sub(cl.files, extra)
2154                         cl.Flush(ui, repo)
2155                 if not cl.files:
2156                         if not cl.copied_from:
2157                                 ui.warn("CL %s has no files; delete (abandon) with hg change -d %s\n" % (cl.name, cl.name))
2158                         else:
2159                                 ui.warn("CL %s has no files; delete locally with hg change -D %s\n" % (cl.name, cl.name))
2160         return 0
2161
2162 #######################################################################
2163 # hg upload
2164
2165 @hgcommand
2166 def upload(ui, repo, name, **opts):
2167         """upload diffs to the code review server
2168
2169         Uploads the current modifications for a given change to the server.
2170         """
2171         if codereview_disabled:
2172                 raise hg_util.Abort(codereview_disabled)
2173
2174         repo.ui.quiet = True
2175         cl, err = LoadCL(ui, repo, name, web=True)
2176         if err != "":
2177                 raise hg_util.Abort(err)
2178         if not cl.local:
2179                 raise hg_util.Abort("cannot upload non-local change")
2180         cl.Upload(ui, repo)
2181         print "%s%s\n" % (server_url_base, cl.name)
2182         return 0
2183
2184 #######################################################################
2185 # Table of commands, supplied to Mercurial for installation.
2186
2187 review_opts = [
2188         ('r', 'reviewer', '', 'add reviewer'),
2189         ('', 'cc', '', 'add cc'),
2190         ('', 'tbr', '', 'add future reviewer'),
2191         ('m', 'message', '', 'change description (for new change)'),
2192 ]
2193
2194 cmdtable = {
2195         # The ^ means to show this command in the help text that
2196         # is printed when running hg with no arguments.
2197         "^change": (
2198                 change,
2199                 [
2200                         ('d', 'delete', None, 'delete existing change list'),
2201                         ('D', 'deletelocal', None, 'delete locally, but do not change CL on server'),
2202                         ('i', 'stdin', None, 'read change list from standard input'),
2203                         ('o', 'stdout', None, 'print change list to standard output'),
2204                         ('p', 'pending', None, 'print pending summary to standard output'),
2205                 ],
2206                 "[-d | -D] [-i] [-o] change# or FILE ..."
2207         ),
2208         "^clpatch": (
2209                 clpatch,
2210                 [
2211                         ('', 'ignore_hgapplydiff_failure', None, 'create CL metadata even if hgapplydiff fails'),
2212                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2213                 ],
2214                 "change#"
2215         ),
2216         # Would prefer to call this codereview-login, but then
2217         # hg help codereview prints the help for this command
2218         # instead of the help for the extension.
2219         "code-login": (
2220                 code_login,
2221                 [],
2222                 "",
2223         ),
2224         "^download": (
2225                 download,
2226                 [],
2227                 "change#"
2228         ),
2229         "^file": (
2230                 file,
2231                 [
2232                         ('d', 'delete', None, 'delete files from change list (but not repository)'),
2233                 ],
2234                 "[-d] change# FILE ..."
2235         ),
2236         "^gofmt": (
2237                 gofmt,
2238                 [
2239                         ('l', 'list', None, 'list files that would change, but do not edit them'),
2240                 ],
2241                 "FILE ..."
2242         ),
2243         "^pending|p": (
2244                 pending,
2245                 [
2246                         ('s', 'short', False, 'show short result form'),
2247                         ('', 'quick', False, 'do not consult codereview server'),
2248                 ],
2249                 "[FILE ...]"
2250         ),
2251         "^ps": (
2252                 ps,
2253                 [],
2254                 "[FILE ...]"
2255         ),
2256         "^pq": (
2257                 pq,
2258                 [],
2259                 "[FILE ...]"
2260         ),
2261         "^mail": (
2262                 mail,
2263                 review_opts + [
2264                 ] + hg_commands.walkopts,
2265                 "[-r reviewer] [--cc cc] [change# | file ...]"
2266         ),
2267         "^release-apply": (
2268                 release_apply,
2269                 [
2270                         ('', 'ignore_hgapplydiff_failure', None, 'create CL metadata even if hgapplydiff fails'),
2271                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2272                 ],
2273                 "change#"
2274         ),
2275         # TODO: release-start, release-tag, weekly-tag
2276         "^submit": (
2277                 submit,
2278                 review_opts + [
2279                         ('', 'no_incoming', None, 'disable initial incoming check (for testing)'),
2280                 ] + hg_commands.walkopts + hg_commands.commitopts + hg_commands.commitopts2,
2281                 "[-r reviewer] [--cc cc] [change# | file ...]"
2282         ),
2283         "^sync": (
2284                 sync,
2285                 [
2286                         ('', 'local', None, 'do not pull changes from remote repository')
2287                 ],
2288                 "[--local]",
2289         ),
2290         "^undo": (
2291                 undo,
2292                 [
2293                         ('', 'ignore_hgapplydiff_failure', None, 'create CL metadata even if hgapplydiff fails'),
2294                         ('', 'no_incoming', None, 'disable check for incoming changes'),
2295                 ],
2296                 "change#"
2297         ),
2298         "^upload": (
2299                 upload,
2300                 [],
2301                 "change#"
2302         ),
2303 }
2304
2305 #######################################################################
2306 # Mercurial extension initialization
2307
2308 def norollback(*pats, **opts):
2309         """(disabled when using this extension)"""
2310         raise hg_util.Abort("codereview extension enabled; use undo instead of rollback")
2311
2312 codereview_init = False
2313
2314 def uisetup(ui):
2315         global testing
2316         testing = ui.config("codereview", "testing")
2317         # Disable the Mercurial commands that might change the repository.
2318         # Only commands in this extension are supposed to do that.
2319         ui.setconfig("hooks", "pre-commit.codereview", precommithook) # runs before 'hg commit'
2320         ui.setconfig("hooks", "precommit.codereview", precommithook) # catches all cases
2321
2322 def reposetup(ui, repo):
2323         global codereview_disabled
2324         global defaultcc
2325
2326         # reposetup gets called both for the local repository
2327         # and also for any repository we are pulling or pushing to.
2328         # Only initialize the first time.
2329         global codereview_init
2330         if codereview_init:
2331                 return
2332         codereview_init = True
2333         start_status_thread()
2334
2335         # Read repository-specific options from lib/codereview/codereview.cfg or codereview.cfg.
2336         root = ''
2337         try:
2338                 root = repo.root
2339         except:
2340                 # Yes, repo might not have root; see issue 959.
2341                 codereview_disabled = 'codereview disabled: repository has no root'
2342                 return
2343
2344         repo_config_path = ''
2345         p1 = root + '/lib/codereview/codereview.cfg'
2346         p2 = root + '/codereview.cfg'
2347         if os.access(p1, os.F_OK):
2348                 repo_config_path = p1
2349         else:
2350                 repo_config_path = p2
2351         try:
2352                 f = open(repo_config_path)
2353                 for line in f:
2354                         if line.startswith('defaultcc:'):
2355                                 defaultcc = SplitCommaSpace(line[len('defaultcc:'):])
2356                         if line.startswith('contributors:'):
2357                                 global contributorsURL
2358                                 contributorsURL = line[len('contributors:'):].strip()
2359         except:
2360                 codereview_disabled = 'codereview disabled: cannot open ' + repo_config_path
2361                 return
2362
2363         remote = ui.config("paths", "default", "")
2364         if remote.find("://") < 0 and not testing:
2365                 raise hg_util.Abort("codereview: default path '%s' is not a URL" % (remote,))
2366
2367         InstallMatch(ui, repo)
2368         RietveldSetup(ui, repo)
2369
2370         # Rollback removes an existing commit.  Don't do that either.
2371         global real_rollback
2372         real_rollback = repo.rollback
2373         repo.rollback = norollback
2374
2375
2376 #######################################################################
2377 # Wrappers around upload.py for interacting with Rietveld
2378
2379 from HTMLParser import HTMLParser
2380
2381 # HTML form parser
2382 class FormParser(HTMLParser):
2383         def __init__(self):
2384                 self.map = {}
2385                 self.curtag = None
2386                 self.curdata = None
2387                 HTMLParser.__init__(self)
2388         def handle_starttag(self, tag, attrs):
2389                 if tag == "input":
2390                         key = None
2391                         value = ''
2392                         for a in attrs:
2393                                 if a[0] == 'name':
2394                                         key = a[1]
2395                                 if a[0] == 'value':
2396                                         value = a[1]
2397                         if key is not None:
2398                                 self.map[key] = value
2399                 if tag == "textarea":
2400                         key = None
2401                         for a in attrs:
2402                                 if a[0] == 'name':
2403                                         key = a[1]
2404                         if key is not None:
2405                                 self.curtag = key
2406                                 self.curdata = ''
2407         def handle_endtag(self, tag):
2408                 if tag == "textarea" and self.curtag is not None:
2409                         self.map[self.curtag] = self.curdata
2410                         self.curtag = None
2411                         self.curdata = None
2412         def handle_charref(self, name):
2413                 self.handle_data(unichr(int(name)))
2414         def handle_entityref(self, name):
2415                 import htmlentitydefs
2416                 if name in htmlentitydefs.entitydefs:
2417                         self.handle_data(htmlentitydefs.entitydefs[name])
2418                 else:
2419                         self.handle_data("&" + name + ";")
2420         def handle_data(self, data):
2421                 if self.curdata is not None:
2422                         self.curdata += data
2423
2424 def JSONGet(ui, path):
2425         try:
2426                 data = MySend(path, force_auth=False)
2427                 typecheck(data, str)
2428                 d = fix_json(json.loads(data))
2429         except:
2430                 ui.warn("JSONGet %s: %s\n" % (path, ExceptionDetail()))
2431                 return None
2432         return d
2433
2434 # Clean up json parser output to match our expectations:
2435 #   * all strings are UTF-8-encoded str, not unicode.
2436 #   * missing fields are missing, not None,
2437 #     so that d.get("foo", defaultvalue) works.
2438 def fix_json(x):
2439         if type(x) in [str, int, float, bool, type(None)]:
2440                 pass
2441         elif type(x) is unicode:
2442                 x = x.encode("utf-8")
2443         elif type(x) is list:
2444                 for i in range(len(x)):
2445                         x[i] = fix_json(x[i])
2446         elif type(x) is dict:
2447                 todel = []
2448                 for k in x:
2449                         if x[k] is None:
2450                                 todel.append(k)
2451                         else:
2452                                 x[k] = fix_json(x[k])
2453                 for k in todel:
2454                         del x[k]
2455         else:
2456                 raise hg_util.Abort("unknown type " + str(type(x)) + " in fix_json")
2457         if type(x) is str:
2458                 x = x.replace('\r\n', '\n')
2459         return x
2460
2461 def IsRietveldSubmitted(ui, clname, hex):
2462         dict = JSONGet(ui, "/api/" + clname + "?messages=true")
2463         if dict is None:
2464                 return False
2465         for msg in dict.get("messages", []):
2466                 text = msg.get("text", "")
2467                 regex = '\*\*\* Submitted as [^*]*?r=([0-9a-f]+)[^ ]* \*\*\*'
2468                 if testing:
2469                         regex = '\*\*\* Submitted as ([0-9a-f]+) \*\*\*'
2470                 m = re.match(regex, text)
2471                 if m is not None and len(m.group(1)) >= 8 and hex.startswith(m.group(1)):
2472                         return True
2473         return False
2474
2475 def IsRietveldMailed(cl):
2476         for msg in cl.dict.get("messages", []):
2477                 if msg.get("text", "").find("I'd like you to review this change") >= 0:
2478                         return True
2479         return False
2480
2481 def DownloadCL(ui, repo, clname):
2482         set_status("downloading CL " + clname)
2483         cl, err = LoadCL(ui, repo, clname, web=True)
2484         if err != "":
2485                 return None, None, None, "error loading CL %s: %s" % (clname, err)
2486
2487         # Find most recent diff
2488         diffs = cl.dict.get("patchsets", [])
2489         if not diffs:
2490                 return None, None, None, "CL has no patch sets"
2491         patchid = diffs[-1]
2492
2493         patchset = JSONGet(ui, "/api/" + clname + "/" + str(patchid))
2494         if patchset is None:
2495                 return None, None, None, "error loading CL patchset %s/%d" % (clname, patchid)
2496         if patchset.get("patchset", 0) != patchid:
2497                 return None, None, None, "malformed patchset information"
2498
2499         vers = ""
2500         msg = patchset.get("message", "").split()
2501         if len(msg) >= 3 and msg[0] == "diff" and msg[1] == "-r":
2502                 vers = msg[2]
2503         diff = "/download/issue" + clname + "_" + str(patchid) + ".diff"
2504
2505         diffdata = MySend(diff, force_auth=False)
2506
2507         # Print warning if email is not in CONTRIBUTORS file.
2508         email = cl.dict.get("owner_email", "")
2509         if not email:
2510                 return None, None, None, "cannot find owner for %s" % (clname)
2511         him = FindContributor(ui, repo, email)
2512         me = FindContributor(ui, repo, None)
2513         if him == me:
2514                 cl.mailed = IsRietveldMailed(cl)
2515         else:
2516                 cl.copied_from = email
2517
2518         return cl, vers, diffdata, ""
2519
2520 def MySend(request_path, payload=None,
2521                 content_type="application/octet-stream",
2522                 timeout=None, force_auth=True,
2523                 **kwargs):
2524         """Run MySend1 maybe twice, because Rietveld is unreliable."""
2525         try:
2526                 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2527         except Exception, e:
2528                 if type(e) != urllib2.HTTPError or e.code != 500:       # only retry on HTTP 500 error
2529                         raise
2530                 print >>sys.stderr, "Loading "+request_path+": "+ExceptionDetail()+"; trying again in 2 seconds."
2531                 time.sleep(2)
2532                 return MySend1(request_path, payload, content_type, timeout, force_auth, **kwargs)
2533
2534 # Like upload.py Send but only authenticates when the
2535 # redirect is to www.google.com/accounts.  This keeps
2536 # unnecessary redirects from happening during testing.
2537 def MySend1(request_path, payload=None,
2538                                 content_type="application/octet-stream",
2539                                 timeout=None, force_auth=True,
2540                                 **kwargs):
2541         """Sends an RPC and returns the response.
2542
2543         Args:
2544                 request_path: The path to send the request to, eg /api/appversion/create.
2545                 payload: The body of the request, or None to send an empty request.
2546                 content_type: The Content-Type header to use.
2547                 timeout: timeout in seconds; default None i.e. no timeout.
2548                         (Note: for large requests on OS X, the timeout doesn't work right.)
2549                 kwargs: Any keyword arguments are converted into query string parameters.
2550
2551         Returns:
2552                 The response body, as a string.
2553         """
2554         # TODO: Don't require authentication.  Let the server say
2555         # whether it is necessary.
2556         global rpc
2557         if rpc == None:
2558                 rpc = GetRpcServer(upload_options)
2559         self = rpc
2560         if not self.authenticated and force_auth:
2561                 self._Authenticate()
2562         if request_path is None:
2563                 return
2564         if timeout is None:
2565                 timeout = 30 # seconds
2566
2567         old_timeout = socket.getdefaulttimeout()
2568         socket.setdefaulttimeout(timeout)
2569         try:
2570                 tries = 0
2571                 while True:
2572                         tries += 1
2573                         args = dict(kwargs)
2574                         url = "https://%s%s" % (self.host, request_path)
2575                         if testing:
2576                                 url = url.replace("https://", "http://")
2577                         if args:
2578                                 url += "?" + urllib.urlencode(args)
2579                         req = self._CreateRequest(url=url, data=payload)
2580                         req.add_header("Content-Type", content_type)
2581                         try:
2582                                 f = self.opener.open(req)
2583                                 response = f.read()
2584                                 f.close()
2585                                 # Translate \r\n into \n, because Rietveld doesn't.
2586                                 response = response.replace('\r\n', '\n')
2587                                 # who knows what urllib will give us
2588                                 if type(response) == unicode:
2589                                         response = response.encode("utf-8")
2590                                 typecheck(response, str)
2591                                 return response
2592                         except urllib2.HTTPError, e:
2593                                 if tries > 3:
2594                                         raise
2595                                 elif e.code == 401:
2596                                         self._Authenticate()
2597                                 elif e.code == 302:
2598                                         loc = e.info()["location"]
2599                                         if not loc.startswith('https://www.google.com/a') or loc.find('/ServiceLogin') < 0:
2600                                                 return ''
2601                                         self._Authenticate()
2602                                 else:
2603                                         raise
2604         finally:
2605                 socket.setdefaulttimeout(old_timeout)
2606
2607 def GetForm(url):
2608         f = FormParser()
2609         f.feed(ustr(MySend(url)))       # f.feed wants unicode
2610         f.close()
2611         # convert back to utf-8 to restore sanity
2612         m = {}
2613         for k,v in f.map.items():
2614                 m[k.encode("utf-8")] = v.replace("\r\n", "\n").encode("utf-8")
2615         return m
2616
2617 def EditDesc(issue, subject=None, desc=None, reviewers=None, cc=None, closed=False, private=False):
2618         set_status("uploading change to description")
2619         form_fields = GetForm("/" + issue + "/edit")
2620         if subject is not None:
2621                 form_fields['subject'] = subject
2622         if desc is not None:
2623                 form_fields['description'] = desc
2624         if reviewers is not None:
2625                 form_fields['reviewers'] = reviewers
2626         if cc is not None:
2627                 form_fields['cc'] = cc
2628         if closed:
2629                 form_fields['closed'] = "checked"
2630         if private:
2631                 form_fields['private'] = "checked"
2632         ctype, body = EncodeMultipartFormData(form_fields.items(), [])
2633         response = MySend("/" + issue + "/edit", body, content_type=ctype)
2634         if response != "":
2635                 print >>sys.stderr, "Error editing description:\n" + "Sent form: \n", form_fields, "\n", response
2636                 sys.exit(2)
2637
2638 def PostMessage(ui, issue, message, reviewers=None, cc=None, send_mail=True, subject=None):
2639         set_status("uploading message")
2640         form_fields = GetForm("/" + issue + "/publish")
2641         if reviewers is not None:
2642                 form_fields['reviewers'] = reviewers
2643         if cc is not None:
2644                 form_fields['cc'] = cc
2645         if send_mail:
2646                 form_fields['send_mail'] = "checked"
2647         else:
2648                 del form_fields['send_mail']
2649         if subject is not None:
2650                 form_fields['subject'] = subject
2651         form_fields['message'] = message
2652
2653         form_fields['message_only'] = '1'       # Don't include draft comments
2654         if reviewers is not None or cc is not None:
2655                 form_fields['message_only'] = ''        # Must set '' in order to override cc/reviewer
2656         ctype = "applications/x-www-form-urlencoded"
2657         body = urllib.urlencode(form_fields)
2658         response = MySend("/" + issue + "/publish", body, content_type=ctype)
2659         if response != "":
2660                 print response
2661                 sys.exit(2)
2662
2663 class opt(object):
2664         pass
2665
2666 def RietveldSetup(ui, repo):
2667         global force_google_account
2668         global rpc
2669         global server
2670         global server_url_base
2671         global upload_options
2672         global verbosity
2673
2674         if not ui.verbose:
2675                 verbosity = 0
2676
2677         # Config options.
2678         x = ui.config("codereview", "server")
2679         if x is not None:
2680                 server = x
2681
2682         # TODO(rsc): Take from ui.username?
2683         email = None
2684         x = ui.config("codereview", "email")
2685         if x is not None:
2686                 email = x
2687
2688         server_url_base = "https://" + server + "/"
2689         if testing:
2690                 server_url_base = server_url_base.replace("https://", "http://")
2691
2692         force_google_account = ui.configbool("codereview", "force_google_account", False)
2693
2694         upload_options = opt()
2695         upload_options.email = email
2696         upload_options.host = None
2697         upload_options.verbose = 0
2698         upload_options.description = None
2699         upload_options.description_file = None
2700         upload_options.reviewers = None
2701         upload_options.cc = None
2702         upload_options.message = None
2703         upload_options.issue = None
2704         upload_options.download_base = False
2705         upload_options.send_mail = False
2706         upload_options.vcs = None
2707         upload_options.server = server
2708         upload_options.save_cookies = True
2709
2710         if testing:
2711                 upload_options.save_cookies = False
2712                 upload_options.email = "test@example.com"
2713
2714         rpc = None
2715
2716         global releaseBranch
2717         tags = repo.branchmap().keys()
2718         if 'release-branch.go10' in tags:
2719                 # NOTE(rsc): This tags.sort is going to get the wrong
2720                 # answer when comparing release-branch.go9 with
2721                 # release-branch.go10.  It will be a while before we care.
2722                 raise hg_util.Abort('tags.sort needs to be fixed for release-branch.go10')
2723         tags.sort()
2724         for t in tags:
2725                 if t.startswith('release-branch.go'):
2726                         releaseBranch = t
2727
2728 def workbranch(name):
2729         return name == "default" or name.startswith('dev.')
2730
2731 #######################################################################
2732 # http://codereview.appspot.com/static/upload.py, heavily edited.
2733
2734 #!/usr/bin/env python
2735 #
2736 # Copyright 2007 Google Inc.
2737 #
2738 # Licensed under the Apache License, Version 2.0 (the "License");
2739 # you may not use this file except in compliance with the License.
2740 # You may obtain a copy of the License at
2741 #
2742 #       http://www.apache.org/licenses/LICENSE-2.0
2743 #
2744 # Unless required by applicable law or agreed to in writing, software
2745 # distributed under the License is distributed on an "AS IS" BASIS,
2746 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
2747 # See the License for the specific language governing permissions and
2748 # limitations under the License.
2749
2750 """Tool for uploading diffs from a version control system to the codereview app.
2751
2752 Usage summary: upload.py [options] [-- diff_options]
2753
2754 Diff options are passed to the diff command of the underlying system.
2755
2756 Supported version control systems:
2757         Git
2758         Mercurial
2759         Subversion
2760
2761 It is important for Git/Mercurial users to specify a tree/node/branch to diff
2762 against by using the '--rev' option.
2763 """
2764 # This code is derived from appcfg.py in the App Engine SDK (open source),
2765 # and from ASPN recipe #146306.
2766
2767 import cookielib
2768 import getpass
2769 import logging
2770 import mimetypes
2771 import optparse
2772 import os
2773 import re
2774 import socket
2775 import subprocess
2776 import sys
2777 import urllib
2778 import urllib2
2779 import urlparse
2780
2781 # The md5 module was deprecated in Python 2.5.
2782 try:
2783         from hashlib import md5
2784 except ImportError:
2785         from md5 import md5
2786
2787 try:
2788         import readline
2789 except ImportError:
2790         pass
2791
2792 # The logging verbosity:
2793 #  0: Errors only.
2794 #  1: Status messages.
2795 #  2: Info logs.
2796 #  3: Debug logs.
2797 verbosity = 1
2798
2799 # Max size of patch or base file.
2800 MAX_UPLOAD_SIZE = 900 * 1024
2801
2802 # whitelist for non-binary filetypes which do not start with "text/"
2803 # .mm (Objective-C) shows up as application/x-freemind on my Linux box.
2804 TEXT_MIMETYPES = [
2805         'application/javascript',
2806         'application/x-javascript',
2807         'application/x-freemind'
2808 ]
2809
2810 def GetEmail(prompt):
2811         """Prompts the user for their email address and returns it.
2812
2813         The last used email address is saved to a file and offered up as a suggestion
2814         to the user. If the user presses enter without typing in anything the last
2815         used email address is used. If the user enters a new address, it is saved
2816         for next time we prompt.
2817
2818         """
2819         last_email_file_name = os.path.expanduser("~/.last_codereview_email_address")
2820         last_email = ""
2821         if os.path.exists(last_email_file_name):
2822                 try:
2823                         last_email_file = open(last_email_file_name, "r")
2824                         last_email = last_email_file.readline().strip("\n")
2825                         last_email_file.close()
2826                         prompt += " [%s]" % last_email
2827                 except IOError, e:
2828                         pass
2829         email = raw_input(prompt + ": ").strip()
2830         if email:
2831                 try:
2832                         last_email_file = open(last_email_file_name, "w")
2833                         last_email_file.write(email)
2834                         last_email_file.close()
2835                 except IOError, e:
2836                         pass
2837         else:
2838                 email = last_email
2839         return email
2840
2841
2842 def StatusUpdate(msg):
2843         """Print a status message to stdout.
2844
2845         If 'verbosity' is greater than 0, print the message.
2846
2847         Args:
2848                 msg: The string to print.
2849         """
2850         if verbosity > 0:
2851                 print msg
2852
2853
2854 def ErrorExit(msg):
2855         """Print an error message to stderr and exit."""
2856         print >>sys.stderr, msg
2857         sys.exit(1)
2858
2859
2860 class ClientLoginError(urllib2.HTTPError):
2861         """Raised to indicate there was an error authenticating with ClientLogin."""
2862
2863         def __init__(self, url, code, msg, headers, args):
2864                 urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
2865                 self.args = args
2866                 # .reason is now a read-only property based on .msg
2867                 # this means we ignore 'msg', but that seems to work fine.
2868                 self.msg = args["Error"]
2869
2870
2871 class AbstractRpcServer(object):
2872         """Provides a common interface for a simple RPC server."""
2873
2874         def __init__(self, host, auth_function, host_override=None, extra_headers={}, save_cookies=False):
2875                 """Creates a new HttpRpcServer.
2876
2877                 Args:
2878                         host: The host to send requests to.
2879                         auth_function: A function that takes no arguments and returns an
2880                                 (email, password) tuple when called. Will be called if authentication
2881                                 is required.
2882                         host_override: The host header to send to the server (defaults to host).
2883                         extra_headers: A dict of extra headers to append to every request.
2884                         save_cookies: If True, save the authentication cookies to local disk.
2885                                 If False, use an in-memory cookiejar instead.  Subclasses must
2886                                 implement this functionality.  Defaults to False.
2887                 """
2888                 self.host = host
2889                 self.host_override = host_override
2890                 self.auth_function = auth_function
2891                 self.authenticated = False
2892                 self.extra_headers = extra_headers
2893                 self.save_cookies = save_cookies
2894                 self.opener = self._GetOpener()
2895                 if self.host_override:
2896                         logging.info("Server: %s; Host: %s", self.host, self.host_override)
2897                 else:
2898                         logging.info("Server: %s", self.host)
2899
2900         def _GetOpener(self):
2901                 """Returns an OpenerDirector for making HTTP requests.
2902
2903                 Returns:
2904                         A urllib2.OpenerDirector object.
2905                 """
2906                 raise NotImplementedError()
2907
2908         def _CreateRequest(self, url, data=None):
2909                 """Creates a new urllib request."""
2910                 logging.debug("Creating request for: '%s' with payload:\n%s", url, data)
2911                 req = urllib2.Request(url, data=data)
2912                 if self.host_override:
2913                         req.add_header("Host", self.host_override)
2914                 for key, value in self.extra_headers.iteritems():
2915                         req.add_header(key, value)
2916                 return req
2917
2918         def _GetAuthToken(self, email, password):
2919                 """Uses ClientLogin to authenticate the user, returning an auth token.
2920
2921                 Args:
2922                         email:    The user's email address
2923                         password: The user's password
2924
2925                 Raises:
2926                         ClientLoginError: If there was an error authenticating with ClientLogin.
2927                         HTTPError: If there was some other form of HTTP error.
2928
2929                 Returns:
2930                         The authentication token returned by ClientLogin.
2931                 """
2932                 account_type = "GOOGLE"
2933                 if self.host.endswith(".google.com") and not force_google_account:
2934                         # Needed for use inside Google.
2935                         account_type = "HOSTED"
2936                 req = self._CreateRequest(
2937                                 url="https://www.google.com/accounts/ClientLogin",
2938                                 data=urllib.urlencode({
2939                                                 "Email": email,
2940                                                 "Passwd": password,
2941                                                 "service": "ah",
2942                                                 "source": "rietveld-codereview-upload",
2943                                                 "accountType": account_type,
2944                                 }),
2945                 )
2946                 try:
2947                         response = self.opener.open(req)
2948                         response_body = response.read()
2949                         response_dict = dict(x.split("=") for x in response_body.split("\n") if x)
2950                         return response_dict["Auth"]
2951                 except urllib2.HTTPError, e:
2952                         if e.code == 403:
2953                                 body = e.read()
2954                                 response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
2955                                 raise ClientLoginError(req.get_full_url(), e.code, e.msg, e.headers, response_dict)
2956                         else:
2957                                 raise
2958
2959         def _GetAuthCookie(self, auth_token):
2960                 """Fetches authentication cookies for an authentication token.
2961
2962                 Args:
2963                         auth_token: The authentication token returned by ClientLogin.
2964
2965                 Raises:
2966                         HTTPError: If there was an error fetching the authentication cookies.
2967                 """
2968                 # This is a dummy value to allow us to identify when we're successful.
2969                 continue_location = "http://localhost/"
2970                 args = {"continue": continue_location, "auth": auth_token}
2971                 reqUrl = "https://%s/_ah/login?%s" % (self.host, urllib.urlencode(args))
2972                 if testing:
2973                         reqUrl = reqUrl.replace("https://", "http://")
2974                 req = self._CreateRequest(reqUrl)
2975                 try:
2976                         response = self.opener.open(req)
2977                 except urllib2.HTTPError, e:
2978                         response = e
2979                 if (response.code != 302 or
2980                                 response.info()["location"] != continue_location):
2981                         raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, response.headers, response.fp)
2982                 self.authenticated = True
2983
2984         def _Authenticate(self):
2985                 """Authenticates the user.
2986
2987                 The authentication process works as follows:
2988                 1) We get a username and password from the user
2989                 2) We use ClientLogin to obtain an AUTH token for the user
2990                                 (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
2991                 3) We pass the auth token to /_ah/login on the server to obtain an
2992                                 authentication cookie. If login was successful, it tries to redirect
2993                                 us to the URL we provided.
2994
2995                 If we attempt to access the upload API without first obtaining an
2996                 authentication cookie, it returns a 401 response (or a 302) and
2997                 directs us to authenticate ourselves with ClientLogin.
2998                 """
2999                 for i in range(3):
3000                         credentials = self.auth_function()
3001                         try:
3002                                 auth_token = self._GetAuthToken(credentials[0], credentials[1])
3003                         except ClientLoginError, e:
3004                                 if e.msg == "BadAuthentication":
3005                                         print >>sys.stderr, "Invalid username or password."
3006                                         continue
3007                                 if e.msg == "CaptchaRequired":
3008                                         print >>sys.stderr, (
3009                                                 "Please go to\n"
3010                                                 "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
3011                                                 "and verify you are a human.  Then try again.")
3012                                         break
3013                                 if e.msg == "NotVerified":
3014                                         print >>sys.stderr, "Account not verified."
3015                                         break
3016                                 if e.msg == "TermsNotAgreed":
3017                                         print >>sys.stderr, "User has not agreed to TOS."
3018                                         break
3019                                 if e.msg == "AccountDeleted":
3020                                         print >>sys.stderr, "The user account has been deleted."
3021                                         break
3022                                 if e.msg == "AccountDisabled":
3023                                         print >>sys.stderr, "The user account has been disabled."
3024                                         break
3025                                 if e.msg == "ServiceDisabled":
3026                                         print >>sys.stderr, "The user's access to the service has been disabled."
3027                                         break
3028                                 if e.msg == "ServiceUnavailable":
3029                                         print >>sys.stderr, "The service is not available; try again later."
3030                                         break
3031                                 raise
3032                         self._GetAuthCookie(auth_token)
3033                         return
3034
3035         def Send(self, request_path, payload=None,
3036                                         content_type="application/octet-stream",
3037                                         timeout=None,
3038                                         **kwargs):
3039                 """Sends an RPC and returns the response.
3040
3041                 Args:
3042                         request_path: The path to send the request to, eg /api/appversion/create.
3043                         payload: The body of the request, or None to send an empty request.
3044                         content_type: The Content-Type header to use.
3045                         timeout: timeout in seconds; default None i.e. no timeout.
3046                                 (Note: for large requests on OS X, the timeout doesn't work right.)
3047                         kwargs: Any keyword arguments are converted into query string parameters.
3048
3049                 Returns:
3050                         The response body, as a string.
3051                 """
3052                 # TODO: Don't require authentication.  Let the server say
3053                 # whether it is necessary.
3054                 if not self.authenticated:
3055                         self._Authenticate()
3056
3057                 old_timeout = socket.getdefaulttimeout()
3058                 socket.setdefaulttimeout(timeout)
3059                 try:
3060                         tries = 0
3061                         while True:
3062                                 tries += 1
3063                                 args = dict(kwargs)
3064                                 url = "https://%s%s" % (self.host, request_path)
3065                                 if testing:
3066                                         url = url.replace("https://", "http://")
3067                                 if args:
3068                                         url += "?" + urllib.urlencode(args)
3069                                 req = self._CreateRequest(url=url, data=payload)
3070                                 req.add_header("Content-Type", content_type)
3071                                 try:
3072                                         f = self.opener.open(req)
3073                                         response = f.read()
3074                                         f.close()
3075                                         return response
3076                                 except urllib2.HTTPError, e:
3077                                         if tries > 3:
3078                                                 raise
3079                                         elif e.code == 401 or e.code == 302:
3080                                                 self._Authenticate()
3081                                         else:
3082                                                 raise
3083                 finally:
3084                         socket.setdefaulttimeout(old_timeout)
3085
3086
3087 class HttpRpcServer(AbstractRpcServer):
3088         """Provides a simplified RPC-style interface for HTTP requests."""
3089
3090         def _Authenticate(self):
3091                 """Save the cookie jar after authentication."""
3092                 super(HttpRpcServer, self)._Authenticate()
3093                 if self.save_cookies:
3094                         StatusUpdate("Saving authentication cookies to %s" % self.cookie_file)
3095                         self.cookie_jar.save()
3096
3097         def _GetOpener(self):
3098                 """Returns an OpenerDirector that supports cookies and ignores redirects.
3099
3100                 Returns:
3101                         A urllib2.OpenerDirector object.
3102                 """
3103                 opener = urllib2.OpenerDirector()
3104                 opener.add_handler(urllib2.ProxyHandler())
3105                 opener.add_handler(urllib2.UnknownHandler())
3106                 opener.add_handler(urllib2.HTTPHandler())
3107                 opener.add_handler(urllib2.HTTPDefaultErrorHandler())
3108                 opener.add_handler(urllib2.HTTPSHandler())
3109                 opener.add_handler(urllib2.HTTPErrorProcessor())
3110                 if self.save_cookies:
3111                         self.cookie_file = os.path.expanduser("~/.codereview_upload_cookies_" + server)
3112                         self.cookie_jar = cookielib.MozillaCookieJar(self.cookie_file)
3113                         if os.path.exists(self.cookie_file):
3114                                 try:
3115                                         self.cookie_jar.load()
3116                                         self.authenticated = True
3117                                         StatusUpdate("Loaded authentication cookies from %s" % self.cookie_file)
3118                                 except (cookielib.LoadError, IOError):
3119                                         # Failed to load cookies - just ignore them.
3120                                         pass
3121                         else:
3122                                 # Create an empty cookie file with mode 600
3123                                 fd = os.open(self.cookie_file, os.O_CREAT, 0600)
3124                                 os.close(fd)
3125                         # Always chmod the cookie file
3126                         os.chmod(self.cookie_file, 0600)
3127                 else:
3128                         # Don't save cookies across runs of update.py.
3129                         self.cookie_jar = cookielib.CookieJar()
3130                 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
3131                 return opener
3132
3133
3134 def GetRpcServer(options):
3135         """Returns an instance of an AbstractRpcServer.
3136
3137         Returns:
3138                 A new AbstractRpcServer, on which RPC calls can be made.
3139         """
3140
3141         rpc_server_class = HttpRpcServer
3142
3143         def GetUserCredentials():
3144                 """Prompts the user for a username and password."""
3145                 # Disable status prints so they don't obscure the password prompt.
3146                 global global_status
3147                 st = global_status
3148                 global_status = None
3149
3150                 email = options.email
3151                 if email is None:
3152                         email = GetEmail("Email (login for uploading to %s)" % options.server)
3153                 password = getpass.getpass("Password for %s: " % email)
3154
3155                 # Put status back.
3156                 global_status = st
3157                 return (email, password)
3158
3159         # If this is the dev_appserver, use fake authentication.
3160         host = (options.host or options.server).lower()
3161         if host == "localhost" or host.startswith("localhost:"):
3162                 email = options.email
3163                 if email is None:
3164                         email = "test@example.com"
3165                         logging.info("Using debug user %s.  Override with --email" % email)
3166                 server = rpc_server_class(
3167                                 options.server,
3168                                 lambda: (email, "password"),
3169                                 host_override=options.host,
3170                                 extra_headers={"Cookie": 'dev_appserver_login="%s:False"' % email},
3171                                 save_cookies=options.save_cookies)
3172                 # Don't try to talk to ClientLogin.
3173                 server.authenticated = True
3174                 return server
3175
3176         return rpc_server_class(options.server, GetUserCredentials,
3177                 host_override=options.host, save_cookies=options.save_cookies)
3178
3179
3180 def EncodeMultipartFormData(fields, files):
3181         """Encode form fields for multipart/form-data.
3182
3183         Args:
3184                 fields: A sequence of (name, value) elements for regular form fields.
3185                 files: A sequence of (name, filename, value) elements for data to be
3186                                         uploaded as files.
3187         Returns:
3188                 (content_type, body) ready for httplib.HTTP instance.
3189
3190         Source:
3191                 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306
3192         """
3193         BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
3194         CRLF = '\r\n'
3195         lines = []
3196         for (key, value) in fields:
3197                 typecheck(key, str)
3198                 typecheck(value, str)
3199                 lines.append('--' + BOUNDARY)
3200                 lines.append('Content-Disposition: form-data; name="%s"' % key)
3201                 lines.append('')
3202                 lines.append(value)
3203         for (key, filename, value) in files:
3204                 typecheck(key, str)
3205                 typecheck(filename, str)
3206                 typecheck(value, str)
3207                 lines.append('--' + BOUNDARY)
3208                 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
3209                 lines.append('Content-Type: %s' % GetContentType(filename))
3210                 lines.append('')
3211                 lines.append(value)
3212         lines.append('--' + BOUNDARY + '--')
3213         lines.append('')
3214         body = CRLF.join(lines)
3215         content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
3216         return content_type, body
3217
3218
3219 def GetContentType(filename):
3220         """Helper to guess the content-type from the filename."""
3221         return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
3222
3223
3224 # Use a shell for subcommands on Windows to get a PATH search.
3225 use_shell = sys.platform.startswith("win")
3226
3227 def RunShellWithReturnCode(command, print_output=False,
3228                 universal_newlines=True, env=os.environ):
3229         """Executes a command and returns the output from stdout and the return code.
3230
3231         Args:
3232                 command: Command to execute.
3233                 print_output: If True, the output is printed to stdout.
3234                         If False, both stdout and stderr are ignored.
3235                 universal_newlines: Use universal_newlines flag (default: True).
3236
3237         Returns:
3238                 Tuple (output, return code)
3239         """
3240         logging.info("Running %s", command)
3241         p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
3242                 shell=use_shell, universal_newlines=universal_newlines, env=env)
3243         if print_output:
3244                 output_array = []
3245                 while True:
3246                         line = p.stdout.readline()
3247                         if not line:
3248                                 break
3249                         print line.strip("\n")
3250                         output_array.append(line)
3251                 output = "".join(output_array)
3252         else:
3253                 output = p.stdout.read()
3254         p.wait()
3255         errout = p.stderr.read()
3256         if print_output and errout:
3257                 print >>sys.stderr, errout
3258         p.stdout.close()
3259         p.stderr.close()
3260         return output, p.returncode
3261
3262
3263 def RunShell(command, silent_ok=False, universal_newlines=True,
3264                 print_output=False, env=os.environ):
3265         data, retcode = RunShellWithReturnCode(command, print_output, universal_newlines, env)
3266         if retcode:
3267                 ErrorExit("Got error status from %s:\n%s" % (command, data))
3268         if not silent_ok and not data:
3269                 ErrorExit("No output from %s" % command)
3270         return data
3271
3272
3273 class VersionControlSystem(object):
3274         """Abstract base class providing an interface to the VCS."""
3275
3276         def __init__(self, options):
3277                 """Constructor.
3278
3279                 Args:
3280                         options: Command line options.
3281                 """
3282                 self.options = options
3283
3284         def GenerateDiff(self, args):
3285                 """Return the current diff as a string.
3286
3287                 Args:
3288                         args: Extra arguments to pass to the diff command.
3289                 """
3290                 raise NotImplementedError(
3291                                 "abstract method -- subclass %s must override" % self.__class__)
3292
3293         def GetUnknownFiles(self):
3294                 """Return a list of files unknown to the VCS."""
3295                 raise NotImplementedError(
3296                                 "abstract method -- subclass %s must override" % self.__class__)
3297
3298         def CheckForUnknownFiles(self):
3299                 """Show an "are you sure?" prompt if there are unknown files."""
3300                 unknown_files = self.GetUnknownFiles()
3301                 if unknown_files:
3302                         print "The following files are not added to version control:"
3303                         for line in unknown_files:
3304                                 print line
3305                         prompt = "Are you sure to continue?(y/N) "
3306                         answer = raw_input(prompt).strip()
3307                         if answer != "y":
3308                                 ErrorExit("User aborted")
3309
3310         def GetBaseFile(self, filename):
3311                 """Get the content of the upstream version of a file.
3312
3313                 Returns:
3314                         A tuple (base_content, new_content, is_binary, status)
3315                                 base_content: The contents of the base file.
3316                                 new_content: For text files, this is empty.  For binary files, this is
3317                                         the contents of the new file, since the diff output won't contain
3318                                         information to reconstruct the current file.
3319                                 is_binary: True iff the file is binary.
3320                                 status: The status of the file.
3321                 """
3322
3323                 raise NotImplementedError(
3324                                 "abstract method -- subclass %s must override" % self.__class__)
3325
3326
3327         def GetBaseFiles(self, diff):
3328                 """Helper that calls GetBase file for each file in the patch.
3329
3330                 Returns:
3331                         A dictionary that maps from filename to GetBaseFile's tuple.  Filenames
3332                         are retrieved based on lines that start with "Index:" or
3333                         "Property changes on:".
3334                 """
3335                 files = {}
3336                 for line in diff.splitlines(True):
3337                         if line.startswith('Index:') or line.startswith('Property changes on:'):
3338                                 unused, filename = line.split(':', 1)
3339                                 # On Windows if a file has property changes its filename uses '\'
3340                                 # instead of '/'.
3341                                 filename = to_slash(filename.strip())
3342                                 files[filename] = self.GetBaseFile(filename)
3343                 return files
3344
3345
3346         def UploadBaseFiles(self, issue, rpc_server, patch_list, patchset, options,
3347                                                                                         files):
3348                 """Uploads the base files (and if necessary, the current ones as well)."""
3349
3350                 def UploadFile(filename, file_id, content, is_binary, status, is_base):
3351                         """Uploads a file to the server."""
3352                         set_status("uploading " + filename)
3353                         file_too_large = False
3354                         if is_base:
3355                                 type = "base"
3356                         else:
3357                                 type = "current"
3358                         if len(content) > MAX_UPLOAD_SIZE:
3359                                 print ("Not uploading the %s file for %s because it's too large." %
3360                                                         (type, filename))
3361                                 file_too_large = True
3362                                 content = ""
3363                         checksum = md5(content).hexdigest()
3364                         if options.verbose > 0 and not file_too_large:
3365                                 print "Uploading %s file for %s" % (type, filename)
3366                         url = "/%d/upload_content/%d/%d" % (int(issue), int(patchset), file_id)
3367                         form_fields = [
3368                                 ("filename", filename),
3369                                 ("status", status),
3370                                 ("checksum", checksum),
3371                                 ("is_binary", str(is_binary)),
3372                                 ("is_current", str(not is_base)),
3373                         ]
3374                         if file_too_large:
3375                                 form_fields.append(("file_too_large", "1"))
3376                         if options.email:
3377                                 form_fields.append(("user", options.email))
3378                         ctype, body = EncodeMultipartFormData(form_fields, [("data", filename, content)])
3379                         response_body = rpc_server.Send(url, body, content_type=ctype)
3380                         if not response_body.startswith("OK"):
3381                                 StatusUpdate("  --> %s" % response_body)
3382                                 sys.exit(1)
3383
3384                 # Don't want to spawn too many threads, nor do we want to
3385                 # hit Rietveld too hard, or it will start serving 500 errors.
3386                 # When 8 works, it's no better than 4, and sometimes 8 is
3387                 # too many for Rietveld to handle.
3388                 MAX_PARALLEL_UPLOADS = 4
3389
3390                 sema = threading.BoundedSemaphore(MAX_PARALLEL_UPLOADS)
3391                 upload_threads = []
3392                 finished_upload_threads = []
3393
3394                 class UploadFileThread(threading.Thread):
3395                         def __init__(self, args):
3396                                 threading.Thread.__init__(self)
3397                                 self.args = args
3398                         def run(self):
3399                                 UploadFile(*self.args)
3400                                 finished_upload_threads.append(self)
3401                                 sema.release()
3402
3403                 def StartUploadFile(*args):
3404                         sema.acquire()
3405                         while len(finished_upload_threads) > 0:
3406                                 t = finished_upload_threads.pop()
3407                                 upload_threads.remove(t)
3408                                 t.join()
3409                         t = UploadFileThread(args)
3410                         upload_threads.append(t)
3411                         t.start()
3412
3413                 def WaitForUploads():
3414                         for t in upload_threads:
3415                                 t.join()
3416
3417                 patches = dict()
3418                 [patches.setdefault(v, k) for k, v in patch_list]
3419                 for filename in patches.keys():
3420                         base_content, new_content, is_binary, status = files[filename]
3421                         file_id_str = patches.get(filename)
3422                         if file_id_str.find("nobase") != -1:
3423                                 base_content = None
3424                                 file_id_str = file_id_str[file_id_str.rfind("_") + 1:]
3425                         file_id = int(file_id_str)
3426                         if base_content != None:
3427                                 StartUploadFile(filename, file_id, base_content, is_binary, status, True)
3428                         if new_content != None:
3429                                 StartUploadFile(filename, file_id, new_content, is_binary, status, False)
3430                 WaitForUploads()
3431
3432         def IsImage(self, filename):
3433                 """Returns true if the filename has an image extension."""
3434                 mimetype =  mimetypes.guess_type(filename)[0]
3435                 if not mimetype:
3436                         return False
3437                 return mimetype.startswith("image/")
3438
3439         def IsBinary(self, filename):
3440                 """Returns true if the guessed mimetyped isnt't in text group."""
3441                 mimetype = mimetypes.guess_type(filename)[0]
3442                 if not mimetype:
3443                         return False  # e.g. README, "real" binaries usually have an extension
3444                 # special case for text files which don't start with text/
3445                 if mimetype in TEXT_MIMETYPES:
3446                         return False
3447                 return not mimetype.startswith("text/")
3448
3449
3450 class FakeMercurialUI(object):
3451         def __init__(self):
3452                 self.quiet = True
3453                 self.output = ''
3454                 self.debugflag = False
3455
3456         def write(self, *args, **opts):
3457                 self.output += ' '.join(args)
3458         def copy(self):
3459                 return self
3460         def status(self, *args, **opts):
3461                 pass
3462
3463         def formatter(self, topic, opts):
3464                 from mercurial.formatter import plainformatter
3465                 return plainformatter(self, topic, opts)
3466
3467         def readconfig(self, *args, **opts):
3468                 pass
3469         def expandpath(self, *args, **opts):
3470                 return global_ui.expandpath(*args, **opts)
3471         def configitems(self, *args, **opts):
3472                 return global_ui.configitems(*args, **opts)
3473         def config(self, *args, **opts):
3474                 return global_ui.config(*args, **opts)
3475
3476 use_hg_shell = False    # set to True to shell out to hg always; slower
3477
3478 class MercurialVCS(VersionControlSystem):
3479         """Implementation of the VersionControlSystem interface for Mercurial."""
3480
3481         def __init__(self, options, ui, repo):
3482                 super(MercurialVCS, self).__init__(options)
3483                 self.ui = ui
3484                 self.repo = repo
3485                 self.status = None
3486                 # Absolute path to repository (we can be in a subdir)
3487                 self.repo_dir = os.path.normpath(repo.root)
3488                 # Compute the subdir
3489                 cwd = os.path.normpath(os.getcwd())
3490                 assert cwd.startswith(self.repo_dir)
3491                 self.subdir = cwd[len(self.repo_dir):].lstrip(r"\/")
3492                 mqparent, err = RunShellWithReturnCode(['hg', 'log', '--rev', 'qparent', '--template={node}'])
3493                 if not err and mqparent != "":
3494                         self.base_rev = mqparent
3495                 else:
3496                         out = RunShell(["hg", "parents", "-q", "--template={node} {branch}"], silent_ok=True).strip()
3497                         if not out:
3498                                 # No revisions; use 0 to mean a repository with nothing.
3499                                 out = "0:0 default"
3500
3501                         # Find parent along current branch.
3502                         branch = repo[None].branch()
3503                         base = ""
3504                         for line in out.splitlines():
3505                                 fields = line.strip().split(' ')
3506                                 if fields[1] == branch:
3507                                         base = fields[0]
3508                                         break
3509                         if base == "":
3510                                 # Use the first parent
3511                                 base = out.strip().split(' ')[0]
3512                         self.base_rev = base
3513
3514         def _GetRelPath(self, filename):
3515                 """Get relative path of a file according to the current directory,
3516                 given its logical path in the repo."""
3517                 assert filename.startswith(self.subdir), (filename, self.subdir)
3518                 return filename[len(self.subdir):].lstrip(r"\/")
3519
3520         def GenerateDiff(self, extra_args):
3521                 # If no file specified, restrict to the current subdir
3522                 extra_args = extra_args or ["."]
3523                 cmd = ["hg", "diff", "--git", "-r", self.base_rev] + extra_args
3524                 data = RunShell(cmd, silent_ok=True)
3525                 svndiff = []
3526                 filecount = 0
3527                 for line in data.splitlines():
3528                         m = re.match("diff --git a/(\S+) b/(\S+)", line)
3529                         if m:
3530                                 # Modify line to make it look like as it comes from svn diff.
3531                                 # With this modification no changes on the server side are required
3532                                 # to make upload.py work with Mercurial repos.
3533                                 # NOTE: for proper handling of moved/copied files, we have to use
3534                                 # the second filename.
3535                                 filename = m.group(2)
3536                                 svndiff.append("Index: %s" % filename)
3537                                 svndiff.append("=" * 67)
3538                                 filecount += 1
3539                                 logging.info(line)
3540                         else:
3541                                 svndiff.append(line)
3542                 if not filecount:
3543                         ErrorExit("No valid patches found in output from hg diff")
3544                 return "\n".join(svndiff) + "\n"
3545
3546         def GetUnknownFiles(self):
3547                 """Return a list of files unknown to the VCS."""
3548                 args = []
3549                 status = RunShell(["hg", "status", "--rev", self.base_rev, "-u", "."],
3550                                 silent_ok=True)
3551                 unknown_files = []
3552                 for line in status.splitlines():
3553                         st, fn = line.split(" ", 1)
3554                         if st == "?":
3555                                 unknown_files.append(fn)
3556                 return unknown_files
3557
3558         def get_hg_status(self, rev, path):
3559                 # We'd like to use 'hg status -C path', but that is buggy
3560                 # (see http://mercurial.selenic.com/bts/issue3023).
3561                 # Instead, run 'hg status -C' without a path
3562                 # and skim the output for the path we want.
3563                 if self.status is None:
3564                         if use_hg_shell:
3565                                 out = RunShell(["hg", "status", "-C", "--rev", rev])
3566                         else:
3567                                 fui = FakeMercurialUI()
3568                                 ret = hg_commands.status(fui, self.repo, *[], **{'rev': [rev], 'copies': True})
3569                                 if ret:
3570                                         raise hg_util.Abort(ret)
3571                                 out = fui.output
3572                         self.status = out.splitlines()
3573                 for i in range(len(self.status)):
3574                         # line is
3575                         #       A path
3576                         #       M path
3577                         # etc
3578                         line = to_slash(self.status[i])
3579                         if line[2:] == path:
3580                                 if i+1 < len(self.status) and self.status[i+1][:2] == '  ':
3581                                         return self.status[i:i+2]
3582                                 return self.status[i:i+1]
3583                 raise hg_util.Abort("no status for " + path)
3584
3585         def GetBaseFile(self, filename):
3586                 set_status("inspecting " + filename)
3587                 # "hg status" and "hg cat" both take a path relative to the current subdir
3588                 # rather than to the repo root, but "hg diff" has given us the full path
3589                 # to the repo root.
3590                 base_content = ""
3591                 new_content = None
3592                 is_binary = False
3593                 oldrelpath = relpath = self._GetRelPath(filename)
3594                 out = self.get_hg_status(self.base_rev, relpath)
3595                 status, what = out[0].split(' ', 1)
3596                 if len(out) > 1 and status == "A" and what == relpath:
3597                         oldrelpath = out[1].strip()
3598                         status = "M"
3599                 if ":" in self.base_rev:
3600                         base_rev = self.base_rev.split(":", 1)[0]
3601                 else:
3602                         base_rev = self.base_rev
3603                 if status != "A":
3604                         if use_hg_shell:
3605                                 base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath], silent_ok=True)
3606                         else:
3607                                 try:
3608                                         base_content = str(self.repo[base_rev][oldrelpath].data())
3609                                 except Exception:
3610                                         pass
3611                         is_binary = "\0" in base_content  # Mercurial's heuristic
3612                 if status != "R":
3613                         try:
3614                                 new_content = open(relpath, "rb").read()
3615                                 is_binary = is_binary or "\0" in new_content
3616                         except Exception:
3617                                 pass
3618                 if is_binary and base_content and use_hg_shell:
3619                         # Fetch again without converting newlines
3620                         base_content = RunShell(["hg", "cat", "-r", base_rev, oldrelpath],
3621                                 silent_ok=True, universal_newlines=False)
3622                 if not is_binary or not self.IsImage(relpath):
3623                         new_content = None
3624                 return base_content, new_content, is_binary, status
3625
3626
3627 # NOTE: The SplitPatch function is duplicated in engine.py, keep them in sync.
3628 def SplitPatch(data):
3629         """Splits a patch into separate pieces for each file.
3630
3631         Args:
3632                 data: A string containing the output of svn diff.
3633
3634         Returns:
3635                 A list of 2-tuple (filename, text) where text is the svn diff output
3636                         pertaining to filename.
3637         """
3638         patches = []
3639         filename = None
3640         diff = []
3641         for line in data.splitlines(True):
3642                 new_filename = None
3643                 if line.startswith('Index:'):
3644                         unused, new_filename = line.split(':', 1)
3645                         new_filename = new_filename.strip()
3646                 elif line.startswith('Property changes on:'):
3647                         unused, temp_filename = line.split(':', 1)
3648                         # When a file is modified, paths use '/' between directories, however
3649                         # when a property is modified '\' is used on Windows.  Make them the same
3650                         # otherwise the file shows up twice.
3651                         temp_filename = to_slash(temp_filename.strip())
3652                         if temp_filename != filename:
3653                                 # File has property changes but no modifications, create a new diff.
3654                                 new_filename = temp_filename
3655                 if new_filename:
3656                         if filename and diff:
3657                                 patches.append((filename, ''.join(diff)))
3658                         filename = new_filename
3659                         diff = [line]
3660                         continue
3661                 if diff is not None:
3662                         diff.append(line)
3663         if filename and diff:
3664                 patches.append((filename, ''.join(diff)))
3665         return patches
3666
3667
3668 def UploadSeparatePatches(issue, rpc_server, patchset, data, options):
3669         """Uploads a separate patch for each file in the diff output.
3670
3671         Returns a list of [patch_key, filename] for each file.
3672         """
3673         patches = SplitPatch(data)
3674         rv = []
3675         for patch in patches:
3676                 set_status("uploading patch for " + patch[0])
3677                 if len(patch[1]) > MAX_UPLOAD_SIZE:
3678                         print ("Not uploading the patch for " + patch[0] +
3679                                 " because the file is too large.")
3680                         continue
3681                 form_fields = [("filename", patch[0])]
3682                 if not options.download_base:
3683                         form_fields.append(("content_upload", "1"))
3684                 files = [("data", "data.diff", patch[1])]
3685                 ctype, body = EncodeMultipartFormData(form_fields, files)
3686                 url = "/%d/upload_patch/%d" % (int(issue), int(patchset))
3687                 print "Uploading patch for " + patch[0]
3688                 response_body = rpc_server.Send(url, body, content_type=ctype)
3689                 lines = response_body.splitlines()
3690                 if not lines or lines[0] != "OK":
3691                         StatusUpdate("  --> %s" % response_body)
3692                         sys.exit(1)
3693                 rv.append([lines[1], patch[0]])
3694         return rv