]> Cypherpunks.ru repositories - gocheese.git/blob - gocheese.go
Fix digest mismatch error
[gocheese.git] / gocheese.go
1 /*
2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
5
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, version 3 of the License.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 */
18
19 // Python private package repository and caching proxy
20 package main
21
22 import (
23         "bytes"
24         "context"
25         "crypto/sha256"
26         "encoding/hex"
27         "flag"
28         "fmt"
29         "io"
30         "io/ioutil"
31         "log"
32         "net"
33         "net/http"
34         "net/url"
35         "os"
36         "os/signal"
37         "path/filepath"
38         "regexp"
39         "runtime"
40         "strings"
41         "syscall"
42         "time"
43
44         "golang.org/x/net/netutil"
45 )
46
47 const (
48         HTMLBegin = `<!DOCTYPE html>
49 <html>
50   <head>
51     <title>Links for %s</title>
52   </head>
53   <body>
54 `
55         HTMLEnd      = "  </body>\n</html>\n"
56         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
57         SHA256Prefix = "sha256="
58         SHA256Ext    = ".sha256"
59         InternalFlag = ".internal"
60         GPGSigExt    = ".asc"
61         GPGSigAttr   = " data-gpg-sig=true"
62
63         Warranty = `This program is free software: you can redistribute it and/or modify
64 it under the terms of the GNU General Public License as published by
65 the Free Software Foundation, version 3 of the License.
66
67 This program is distributed in the hope that it will be useful,
68 but WITHOUT ANY WARRANTY; without even the implied warranty of
69 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
70 GNU General Public License for more details.
71
72 You should have received a copy of the GNU General Public License
73 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
74 )
75
76 var (
77         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
78         Version string = "UNKNOWN"
79
80         root             = flag.String("root", "./packages", "Path to packages directory")
81         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
82         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
83         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
84         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
85         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
86         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
87         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
88         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
89         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
90         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
91         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
92         version          = flag.Bool("version", false, "Print version information")
93         warranty         = flag.Bool("warranty", false, "Print warranty information")
94
95         killed bool
96
97         normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+")
98 )
99
100 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
101         path := filepath.Join(*root, dir)
102         if _, err := os.Stat(path); os.IsNotExist(err) {
103                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
104                         http.Error(w, err.Error(), http.StatusInternalServerError)
105                         return false
106                 }
107                 log.Println(r.RemoteAddr, "mkdir", dir)
108         }
109         return true
110 }
111
112 func refreshDir(
113         w http.ResponseWriter,
114         r *http.Request,
115         dir,
116         filenameGet string,
117         gpgUpdate bool,
118 ) bool {
119         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
120                 return true
121         }
122         resp, err := http.Get(*pypiURL + dir + "/")
123         if err != nil {
124                 http.Error(w, err.Error(), http.StatusBadGateway)
125                 return false
126         }
127         body, err := ioutil.ReadAll(resp.Body)
128         resp.Body.Close()
129         if err != nil {
130                 http.Error(w, err.Error(), http.StatusBadGateway)
131                 return false
132         }
133         if !mkdirForPkg(w, r, dir) {
134                 return false
135         }
136         dirPath := filepath.Join(*root, dir)
137         var submatches []string
138         var uri string
139         var filename string
140         var path string
141         var pkgURL *url.URL
142         var digest []byte
143         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
144                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
145                 if len(submatches) == 0 {
146                         continue
147                 }
148                 uri = submatches[1]
149                 filename = submatches[2]
150                 if pkgURL, err = url.Parse(uri); err != nil {
151                         http.Error(w, err.Error(), http.StatusInternalServerError)
152                         return false
153                 }
154                 if !strings.HasPrefix(pkgURL.Fragment, SHA256Prefix) {
155                         log.Println(r.RemoteAddr, "pypi", filename, "no SHA256 digest provided")
156                         http.Error(w, "no SHA256 digest provided", http.StatusBadGateway)
157                         return false
158                 }
159                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
160                 if err != nil {
161                         http.Error(w, err.Error(), http.StatusBadGateway)
162                         return false
163                 }
164                 pkgURL.Fragment = ""
165                 uri = pkgURL.String()
166                 if pkgURL.Host == "" {
167                         uri = *pypiURL + strings.TrimPrefix(uri, "/")
168                 }
169                 path = filepath.Join(dirPath, filename)
170                 if filename == filenameGet {
171                         if killed {
172                                 // Skip heavy remote call, when shutting down
173                                 http.Error(w, "shutting down", http.StatusInternalServerError)
174                                 return false
175                         }
176                         log.Println(r.RemoteAddr, "pypi download", filename)
177                         resp, err = http.Get(uri)
178                         if err != nil {
179                                 log.Println(r.RemoteAddr, "pypi download error:", err.Error())
180                                 http.Error(w, err.Error(), http.StatusBadGateway)
181                                 return false
182                         }
183                         defer resp.Body.Close()
184                         hasher := sha256.New()
185                         dst, err := TempFile(dirPath)
186                         if err != nil {
187                                 http.Error(w, err.Error(), http.StatusInternalServerError)
188                                 return false
189                         }
190                         wr := io.MultiWriter(hasher, dst)
191                         if _, err = io.Copy(wr, resp.Body); err != nil {
192                                 os.Remove(dst.Name())
193                                 dst.Close()
194                                 http.Error(w, err.Error(), http.StatusInternalServerError)
195                                 return false
196                         }
197                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
198                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
199                                 os.Remove(dst.Name())
200                                 dst.Close()
201                                 http.Error(w, "digest mismatch", http.StatusBadGateway)
202                                 return false
203                         }
204                         if err = dst.Sync(); err != nil {
205                                 os.Remove(dst.Name())
206                                 dst.Close()
207                                 http.Error(w, err.Error(), http.StatusInternalServerError)
208                                 return false
209                         }
210                         dst.Close()
211                         if err = os.Rename(dst.Name(), path); err != nil {
212                                 http.Error(w, err.Error(), http.StatusInternalServerError)
213                                 return false
214                         }
215                         if err = DirSync(dirPath); err != nil {
216                                 http.Error(w, err.Error(), http.StatusInternalServerError)
217                                 return false
218                         }
219                 }
220                 if filename == filenameGet || gpgUpdate {
221                         if _, err = os.Stat(path); err != nil {
222                                 goto GPGSigSkip
223                         }
224                         resp, err := http.Get(uri + GPGSigExt)
225                         if err != nil {
226                                 goto GPGSigSkip
227                         }
228                         if resp.StatusCode != http.StatusOK {
229                                 resp.Body.Close()
230                                 goto GPGSigSkip
231                         }
232                         sig, err := ioutil.ReadAll(resp.Body)
233                         resp.Body.Close()
234                         if err != nil {
235                                 goto GPGSigSkip
236                         }
237                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
238                                 http.Error(w, err.Error(), http.StatusInternalServerError)
239                                 return false
240                         }
241                         log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
242                 }
243         GPGSigSkip:
244                 path = path + SHA256Ext
245                 _, err = os.Stat(path)
246                 if err == nil {
247                         continue
248                 }
249                 if !os.IsNotExist(err) {
250                         http.Error(w, err.Error(), http.StatusInternalServerError)
251                         return false
252                 }
253                 log.Println(r.RemoteAddr, "pypi touch", filename)
254                 if err = WriteFileSync(dirPath, path, digest); err != nil {
255                         http.Error(w, err.Error(), http.StatusInternalServerError)
256                         return false
257                 }
258         }
259         return true
260 }
261
262 func listRoot(w http.ResponseWriter, r *http.Request) {
263         files, err := ioutil.ReadDir(*root)
264         if err != nil {
265                 http.Error(w, err.Error(), http.StatusInternalServerError)
266                 return
267         }
268         var result bytes.Buffer
269         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
270         for _, file := range files {
271                 if file.Mode().IsDir() {
272                         result.WriteString(fmt.Sprintf(
273                                 HTMLElement,
274                                 *refreshURLPath+file.Name()+"/",
275                                 file.Name(),
276                         ))
277                 }
278         }
279         result.WriteString(HTMLEnd)
280         w.Write(result.Bytes())
281 }
282
283 func listDir(
284         w http.ResponseWriter,
285         r *http.Request,
286         dir string,
287         autorefresh,
288         gpgUpdate bool,
289 ) {
290         dirPath := filepath.Join(*root, dir)
291         if autorefresh {
292                 if !refreshDir(w, r, dir, "", gpgUpdate) {
293                         return
294                 }
295         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) {
296                 return
297         }
298         files, err := ioutil.ReadDir(dirPath)
299         if err != nil {
300                 http.Error(w, err.Error(), http.StatusInternalServerError)
301                 return
302         }
303         var result bytes.Buffer
304         result.WriteString(fmt.Sprintf(HTMLBegin, dir))
305         var data []byte
306         var gpgSigAttr string
307         var filenameClean string
308         for _, file := range files {
309                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
310                         continue
311                 }
312                 if killed {
313                         // Skip expensive I/O when shutting down
314                         http.Error(w, "shutting down", http.StatusInternalServerError)
315                         return
316                 }
317                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
318                 if err != nil {
319                         http.Error(w, err.Error(), http.StatusInternalServerError)
320                         return
321                 }
322                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
323                 if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) {
324                         gpgSigAttr = ""
325                 } else {
326                         gpgSigAttr = GPGSigAttr
327                 }
328                 result.WriteString(fmt.Sprintf(
329                         HTMLElement,
330                         strings.Join([]string{
331                                 *refreshURLPath, dir, "/",
332                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
333                         }, ""),
334                         gpgSigAttr,
335                         filenameClean,
336                 ))
337         }
338         result.WriteString(HTMLEnd)
339         w.Write(result.Bytes())
340 }
341
342 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
343         log.Println(r.RemoteAddr, "get", filename)
344         path := filepath.Join(*root, dir, filename)
345         if _, err := os.Stat(path); os.IsNotExist(err) {
346                 if !refreshDir(w, r, dir, filename, false) {
347                         return
348                 }
349         }
350         http.ServeFile(w, r, path)
351 }
352
353 func serveUpload(w http.ResponseWriter, r *http.Request) {
354         // Authentication
355         username, password, ok := r.BasicAuth()
356         if !ok {
357                 log.Println(r.RemoteAddr, "unauthenticated", username)
358                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
359                 return
360         }
361         auther, ok := passwords[username]
362         if !ok || !auther.Auth(password) {
363                 log.Println(r.RemoteAddr, "unauthenticated", username)
364                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
365                 return
366         }
367
368         // Form parsing
369         var err error
370         if err = r.ParseMultipartForm(1 << 20); err != nil {
371                 http.Error(w, err.Error(), http.StatusBadRequest)
372                 return
373         }
374         pkgNames, exists := r.MultipartForm.Value["name"]
375         if !exists || len(pkgNames) != 1 {
376                 http.Error(w, "single name is expected in request", http.StatusBadRequest)
377                 return
378         }
379         dir := normalizationRe.ReplaceAllString(pkgNames[0], "-")
380         dirPath := filepath.Join(*root, dir)
381         var digestExpected []byte
382         if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists {
383                 digestExpected, err = hex.DecodeString(digestExpectedHex[0])
384                 if err != nil {
385                         http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest)
386                         return
387                 }
388         }
389         gpgSigsExpected := make(map[string]struct{})
390
391         // Checking is it internal package
392         if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil {
393                 log.Println(r.RemoteAddr, "non-internal package", dir)
394                 http.Error(w, "unknown internal package", http.StatusUnauthorized)
395                 return
396         }
397
398         for _, file := range r.MultipartForm.File["content"] {
399                 filename := file.Filename
400                 gpgSigsExpected[filename+GPGSigExt] = struct{}{}
401                 log.Println(r.RemoteAddr, "put", filename, "by", username)
402                 path := filepath.Join(dirPath, filename)
403                 if _, err = os.Stat(path); err == nil {
404                         log.Println(r.RemoteAddr, "already exists", filename)
405                         http.Error(w, "already exists", http.StatusBadRequest)
406                         return
407                 }
408                 if !mkdirForPkg(w, r, dir) {
409                         return
410                 }
411                 src, err := file.Open()
412                 defer src.Close()
413                 if err != nil {
414                         http.Error(w, err.Error(), http.StatusInternalServerError)
415                         return
416                 }
417                 dst, err := TempFile(dirPath)
418                 if err != nil {
419                         http.Error(w, err.Error(), http.StatusInternalServerError)
420                         return
421                 }
422                 hasher := sha256.New()
423                 wr := io.MultiWriter(hasher, dst)
424                 if _, err = io.Copy(wr, src); err != nil {
425                         os.Remove(dst.Name())
426                         dst.Close()
427                         http.Error(w, err.Error(), http.StatusInternalServerError)
428                         return
429                 }
430                 if err = dst.Sync(); err != nil {
431                         os.Remove(dst.Name())
432                         dst.Close()
433                         http.Error(w, err.Error(), http.StatusInternalServerError)
434                         return
435                 }
436                 dst.Close()
437                 digest := hasher.Sum(nil)
438                 if digestExpected != nil {
439                         if bytes.Compare(digestExpected, digest) == 0 {
440                                 log.Println(r.RemoteAddr, filename, "good checksum received")
441                         } else {
442                                 log.Println(r.RemoteAddr, filename, "bad checksum received")
443                                 http.Error(w, "bad checksum", http.StatusBadRequest)
444                                 os.Remove(dst.Name())
445                                 return
446                         }
447                 }
448                 if err = os.Rename(dst.Name(), path); err != nil {
449                         http.Error(w, err.Error(), http.StatusInternalServerError)
450                         return
451                 }
452                 if err = DirSync(dirPath); err != nil {
453                         http.Error(w, err.Error(), http.StatusInternalServerError)
454                         return
455                 }
456                 if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil {
457                         http.Error(w, err.Error(), http.StatusInternalServerError)
458                         return
459                 }
460         }
461         for _, file := range r.MultipartForm.File["gpg_signature"] {
462                 filename := file.Filename
463                 if _, exists := gpgSigsExpected[filename]; !exists {
464                         http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest)
465                         return
466                 }
467                 delete(gpgSigsExpected, filename)
468                 log.Println(r.RemoteAddr, "put", filename, "by", username)
469                 path := filepath.Join(dirPath, filename)
470                 if _, err = os.Stat(path); err == nil {
471                         log.Println(r.RemoteAddr, "already exists", filename)
472                         http.Error(w, "already exists", http.StatusBadRequest)
473                         return
474                 }
475                 src, err := file.Open()
476                 if err != nil {
477                         http.Error(w, err.Error(), http.StatusInternalServerError)
478                         return
479                 }
480                 sig, err := ioutil.ReadAll(src)
481                 src.Close()
482                 if err != nil {
483                         http.Error(w, err.Error(), http.StatusInternalServerError)
484                         return
485                 }
486                 if err = WriteFileSync(dirPath, path, sig); err != nil {
487                         http.Error(w, err.Error(), http.StatusInternalServerError)
488                         return
489                 }
490         }
491 }
492
493 func handler(w http.ResponseWriter, r *http.Request) {
494         switch r.Method {
495         case "GET":
496                 var path string
497                 var autorefresh bool
498                 var gpgUpdate bool
499                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
500                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
501                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
502                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
503                         autorefresh = true
504                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
505                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
506                         autorefresh = true
507                         gpgUpdate = true
508                 } else {
509                         http.Error(w, "unknown action", http.StatusBadRequest)
510                         return
511                 }
512                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
513                 if len(parts) > 2 {
514                         http.Error(w, "invalid path", http.StatusBadRequest)
515                         return
516                 }
517                 if len(parts) == 1 {
518                         if parts[0] == "" {
519                                 listRoot(w, r)
520                         } else {
521                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
522                         }
523                 } else {
524                         servePkg(w, r, parts[0], parts[1])
525                 }
526         case "POST":
527                 serveUpload(w, r)
528         default:
529                 http.Error(w, "unknown action", http.StatusBadRequest)
530         }
531 }
532
533 func goodIntegrity() bool {
534         dirs, err := ioutil.ReadDir(*root)
535         if err != nil {
536                 log.Fatal(err)
537         }
538         hasher := sha256.New()
539         digest := make([]byte, sha256.Size)
540         isGood := true
541         var data []byte
542         var pkgName string
543         for _, dir := range dirs {
544                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
545                 if err != nil {
546                         log.Fatal(err)
547                 }
548                 for _, file := range files {
549                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
550                                 continue
551                         }
552                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
553                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
554                         if err != nil {
555                                 if os.IsNotExist(err) {
556                                         continue
557                                 }
558                                 log.Fatal(err)
559                         }
560                         hasher.Write(data)
561                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
562                         if err != nil {
563                                 log.Fatal(err)
564                         }
565                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
566                                 fmt.Println(pkgName, "GOOD")
567                         } else {
568                                 isGood = false
569                                 fmt.Println(pkgName, "BAD")
570                         }
571                         hasher.Reset()
572                 }
573         }
574         return isGood
575 }
576
577 func main() {
578         flag.Parse()
579         if *warranty {
580                 fmt.Println(Warranty)
581                 return
582         }
583         if *version {
584                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
585                 return
586         }
587         if *fsck {
588                 if !goodIntegrity() {
589                         os.Exit(1)
590                 }
591                 return
592         }
593         if *passwdCheck {
594                 refreshPasswd()
595                 return
596         }
597         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
598                 log.Fatalln("Both -tls-cert and -tls-key are required")
599         }
600         refreshPasswd()
601         log.Println("root:", *root, "bind:", *bind)
602
603         ln, err := net.Listen("tcp", *bind)
604         if err != nil {
605                 log.Fatal(err)
606         }
607         ln = netutil.LimitListener(ln, *maxClients)
608         server := &http.Server{
609                 ReadTimeout:  time.Minute,
610                 WriteTimeout: time.Minute,
611         }
612         http.HandleFunc(*norefreshURLPath, handler)
613         http.HandleFunc(*refreshURLPath, handler)
614         http.HandleFunc(*gpgUpdateURLPath, handler)
615
616         needsRefreshPasswd := make(chan os.Signal, 0)
617         needsShutdown := make(chan os.Signal, 0)
618         exitErr := make(chan error, 0)
619         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
620         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
621         go func() {
622                 for range needsRefreshPasswd {
623                         log.Println("Refreshing passwords")
624                         refreshPasswd()
625                 }
626         }()
627         go func(s *http.Server) {
628                 <-needsShutdown
629                 killed = true
630                 log.Println("Shutting down")
631                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
632                 exitErr <- s.Shutdown(ctx)
633                 cancel()
634         }(server)
635
636         if *tlsCert == "" {
637                 err = server.Serve(ln)
638         } else {
639                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
640         }
641         if err != http.ErrServerClosed {
642                 log.Fatal(err)
643         }
644         if err := <-exitErr; err != nil {
645                 log.Fatal(err)
646         }
647 }