]> Cypherpunks.ru repositories - gocheese.git/blob - gocheese.go
Check for 404 error of .asc downloads
[gocheese.git] / gocheese.go
1 /*
2 GoCheese -- Python private package repository and caching proxy
3 Copyright (C) 2019 Sergey Matveev <stargrave@stargrave.org>
4               2019 Elena Balakhonova <balakhonova_e@riseup.net>
5
6 This program is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, version 3 of the License.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program.  If not, see <http://www.gnu.org/licenses/>.
17 */
18
19 // Python private package repository and caching proxy
20 package main
21
22 import (
23         "bytes"
24         "context"
25         "crypto/sha256"
26         "encoding/hex"
27         "flag"
28         "fmt"
29         "io"
30         "io/ioutil"
31         "log"
32         "net"
33         "net/http"
34         "net/url"
35         "os"
36         "os/signal"
37         "path/filepath"
38         "regexp"
39         "runtime"
40         "strings"
41         "syscall"
42         "time"
43
44         "golang.org/x/net/netutil"
45 )
46
47 const (
48         HTMLBegin = `<!DOCTYPE html>
49 <html>
50   <head>
51     <title>Links for %s</title>
52   </head>
53   <body>
54 `
55         HTMLEnd      = "  </body>\n</html>\n"
56         HTMLElement  = "    <a href=\"%s\"%s>%s</a><br/>\n"
57         SHA256Prefix = "sha256="
58         SHA256Ext    = ".sha256"
59         InternalFlag = ".internal"
60         GPGSigExt    = ".asc"
61         GPGSigAttr   = " data-gpg-sig=true"
62
63         Warranty = `This program is free software: you can redistribute it and/or modify
64 it under the terms of the GNU General Public License as published by
65 the Free Software Foundation, version 3 of the License.
66
67 This program is distributed in the hope that it will be useful,
68 but WITHOUT ANY WARRANTY; without even the implied warranty of
69 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
70 GNU General Public License for more details.
71
72 You should have received a copy of the GNU General Public License
73 along with this program.  If not, see <http://www.gnu.org/licenses/>.`
74 )
75
76 var (
77         pkgPyPI        = regexp.MustCompile(`^.*<a href="([^"]+)"[^>]*>(.+)</a><br/>.*$`)
78         Version string = "UNKNOWN"
79
80         root             = flag.String("root", "./packages", "Path to packages directory")
81         bind             = flag.String("bind", "[::]:8080", "Address to bind to")
82         tlsCert          = flag.String("tls-cert", "", "Path to TLS X.509 certificate")
83         tlsKey           = flag.String("tls-key", "", "Path to TLS X.509 private key")
84         norefreshURLPath = flag.String("norefresh", "/norefresh/", "Non-refreshing URL path")
85         refreshURLPath   = flag.String("refresh", "/simple/", "Auto-refreshing URL path")
86         gpgUpdateURLPath = flag.String("gpgupdate", "/gpgupdate/", "GPG forceful refreshing URL path")
87         pypiURL          = flag.String("pypi", "https://pypi.org/simple/", "Upstream PyPI URL")
88         passwdPath       = flag.String("passwd", "passwd", "Path to file with authenticators")
89         passwdCheck      = flag.Bool("passwd-check", false, "Test the -passwd file for syntax errors and exit")
90         fsck             = flag.Bool("fsck", false, "Check integrity of all packages")
91         maxClients       = flag.Int("maxclients", 128, "Maximal amount of simultaneous clients")
92         version          = flag.Bool("version", false, "Print version information")
93         warranty         = flag.Bool("warranty", false, "Print warranty information")
94
95         killed bool
96
97         normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+")
98 )
99
100 func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool {
101         path := filepath.Join(*root, dir)
102         if _, err := os.Stat(path); os.IsNotExist(err) {
103                 if err = os.Mkdir(path, os.FileMode(0777)); err != nil {
104                         http.Error(w, err.Error(), http.StatusInternalServerError)
105                         return false
106                 }
107                 log.Println(r.RemoteAddr, "mkdir", dir)
108         }
109         return true
110 }
111
112 func refreshDir(
113         w http.ResponseWriter,
114         r *http.Request,
115         dir,
116         filenameGet string,
117         gpgUpdate bool,
118 ) bool {
119         if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil {
120                 return true
121         }
122         resp, err := http.Get(*pypiURL + dir + "/")
123         if err != nil {
124                 http.Error(w, err.Error(), http.StatusBadGateway)
125                 return false
126         }
127         body, err := ioutil.ReadAll(resp.Body)
128         resp.Body.Close()
129         if err != nil {
130                 http.Error(w, err.Error(), http.StatusBadGateway)
131                 return false
132         }
133         if !mkdirForPkg(w, r, dir) {
134                 return false
135         }
136         dirPath := filepath.Join(*root, dir)
137         var submatches []string
138         var uri string
139         var filename string
140         var path string
141         var pkgURL *url.URL
142         var digest []byte
143         for _, lineRaw := range bytes.Split(body, []byte("\n")) {
144                 submatches = pkgPyPI.FindStringSubmatch(string(lineRaw))
145                 if len(submatches) == 0 {
146                         continue
147                 }
148                 uri = submatches[1]
149                 filename = submatches[2]
150                 if pkgURL, err = url.Parse(uri); err != nil {
151                         http.Error(w, err.Error(), http.StatusInternalServerError)
152                         return false
153                 }
154                 digest, err = hex.DecodeString(strings.TrimPrefix(pkgURL.Fragment, SHA256Prefix))
155                 if err != nil {
156                         http.Error(w, err.Error(), http.StatusBadGateway)
157                         return false
158                 }
159                 pkgURL.Fragment = ""
160                 path = filepath.Join(dirPath, filename)
161                 if filename == filenameGet {
162                         if killed {
163                                 // Skip heavy remote call, when shutting down
164                                 http.Error(w, "shutting down", http.StatusInternalServerError)
165                                 return false
166                         }
167                         log.Println(r.RemoteAddr, "pypi download", filename)
168                         resp, err = http.Get(pkgURL.String())
169                         if err != nil {
170                                 http.Error(w, err.Error(), http.StatusBadGateway)
171                                 return false
172                         }
173                         defer resp.Body.Close()
174                         hasher := sha256.New()
175                         dst, err := TempFile(dirPath)
176                         if err != nil {
177                                 http.Error(w, err.Error(), http.StatusInternalServerError)
178                                 return false
179                         }
180                         wr := io.MultiWriter(hasher, dst)
181                         if _, err = io.Copy(wr, resp.Body); err != nil {
182                                 os.Remove(dst.Name())
183                                 dst.Close()
184                                 http.Error(w, err.Error(), http.StatusInternalServerError)
185                                 return false
186                         }
187                         if bytes.Compare(hasher.Sum(nil), digest) != 0 {
188                                 log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch")
189                                 os.Remove(dst.Name())
190                                 dst.Close()
191                                 http.Error(w, err.Error(), http.StatusBadGateway)
192                                 return false
193                         }
194                         if err = dst.Sync(); err != nil {
195                                 os.Remove(dst.Name())
196                                 dst.Close()
197                                 http.Error(w, err.Error(), http.StatusInternalServerError)
198                                 return false
199                         }
200                         dst.Close()
201                         if err = os.Rename(dst.Name(), path); err != nil {
202                                 http.Error(w, err.Error(), http.StatusInternalServerError)
203                                 return false
204                         }
205                         if err = DirSync(dirPath); err != nil {
206                                 http.Error(w, err.Error(), http.StatusInternalServerError)
207                                 return false
208                         }
209                 }
210                 if filename == filenameGet || gpgUpdate {
211                         if _, err = os.Stat(path); err != nil {
212                                 goto GPGSigSkip
213                         }
214                         resp, err := http.Get(pkgURL.String() + GPGSigExt)
215                         if err != nil {
216                                 goto GPGSigSkip
217                         }
218                         if resp.StatusCode != http.StatusOK {
219                                 resp.Body.Close()
220                                 goto GPGSigSkip
221                         }
222                         sig, err := ioutil.ReadAll(resp.Body)
223                         resp.Body.Close()
224                         if err != nil {
225                                 goto GPGSigSkip
226                         }
227                         if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil {
228                                 http.Error(w, err.Error(), http.StatusInternalServerError)
229                                 return false
230                         }
231                         log.Println(r.RemoteAddr, "pypi downloaded signature", filename)
232                 }
233         GPGSigSkip:
234                 path = path + SHA256Ext
235                 _, err = os.Stat(path)
236                 if err == nil {
237                         continue
238                 }
239                 if !os.IsNotExist(err) {
240                         http.Error(w, err.Error(), http.StatusInternalServerError)
241                         return false
242                 }
243                 log.Println(r.RemoteAddr, "pypi touch", filename)
244                 if err = WriteFileSync(dirPath, path, digest); err != nil {
245                         http.Error(w, err.Error(), http.StatusInternalServerError)
246                         return false
247                 }
248         }
249         return true
250 }
251
252 func listRoot(w http.ResponseWriter, r *http.Request) {
253         files, err := ioutil.ReadDir(*root)
254         if err != nil {
255                 http.Error(w, err.Error(), http.StatusInternalServerError)
256                 return
257         }
258         var result bytes.Buffer
259         result.WriteString(fmt.Sprintf(HTMLBegin, "root"))
260         for _, file := range files {
261                 if file.Mode().IsDir() {
262                         result.WriteString(fmt.Sprintf(
263                                 HTMLElement,
264                                 *refreshURLPath+file.Name()+"/",
265                                 file.Name(),
266                         ))
267                 }
268         }
269         result.WriteString(HTMLEnd)
270         w.Write(result.Bytes())
271 }
272
273 func listDir(
274         w http.ResponseWriter,
275         r *http.Request,
276         dir string,
277         autorefresh,
278         gpgUpdate bool,
279 ) {
280         dirPath := filepath.Join(*root, dir)
281         if autorefresh {
282                 if !refreshDir(w, r, dir, "", gpgUpdate) {
283                         return
284                 }
285         } else if _, err := os.Stat(dirPath); os.IsNotExist(err) && !refreshDir(w, r, dir, "", false) {
286                 return
287         }
288         files, err := ioutil.ReadDir(dirPath)
289         if err != nil {
290                 http.Error(w, err.Error(), http.StatusInternalServerError)
291                 return
292         }
293         var result bytes.Buffer
294         result.WriteString(fmt.Sprintf(HTMLBegin, dir))
295         var data []byte
296         var gpgSigAttr string
297         var filenameClean string
298         for _, file := range files {
299                 if !strings.HasSuffix(file.Name(), SHA256Ext) {
300                         continue
301                 }
302                 if killed {
303                         // Skip expensive I/O when shutting down
304                         http.Error(w, "shutting down", http.StatusInternalServerError)
305                         return
306                 }
307                 data, err = ioutil.ReadFile(filepath.Join(dirPath, file.Name()))
308                 if err != nil {
309                         http.Error(w, err.Error(), http.StatusInternalServerError)
310                         return
311                 }
312                 filenameClean = strings.TrimSuffix(file.Name(), SHA256Ext)
313                 if _, err = os.Stat(filepath.Join(dirPath, filenameClean+GPGSigExt)); os.IsNotExist(err) {
314                         gpgSigAttr = ""
315                 } else {
316                         gpgSigAttr = GPGSigAttr
317                 }
318                 result.WriteString(fmt.Sprintf(
319                         HTMLElement,
320                         strings.Join([]string{
321                                 *refreshURLPath, dir, "/",
322                                 filenameClean, "#", SHA256Prefix, hex.EncodeToString(data),
323                         }, ""),
324                         gpgSigAttr,
325                         filenameClean,
326                 ))
327         }
328         result.WriteString(HTMLEnd)
329         w.Write(result.Bytes())
330 }
331
332 func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) {
333         log.Println(r.RemoteAddr, "get", filename)
334         path := filepath.Join(*root, dir, filename)
335         if _, err := os.Stat(path); os.IsNotExist(err) {
336                 if !refreshDir(w, r, dir, filename, false) {
337                         return
338                 }
339         }
340         http.ServeFile(w, r, path)
341 }
342
343 func serveUpload(w http.ResponseWriter, r *http.Request) {
344         // Authentication
345         username, password, ok := r.BasicAuth()
346         if !ok {
347                 log.Println(r.RemoteAddr, "unauthenticated", username)
348                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
349                 return
350         }
351         auther, ok := passwords[username]
352         if !ok || !auther.Auth(password) {
353                 log.Println(r.RemoteAddr, "unauthenticated", username)
354                 http.Error(w, "unauthenticated", http.StatusUnauthorized)
355                 return
356         }
357
358         // Form parsing
359         var err error
360         if err = r.ParseMultipartForm(1 << 20); err != nil {
361                 http.Error(w, err.Error(), http.StatusBadRequest)
362                 return
363         }
364         pkgNames, exists := r.MultipartForm.Value["name"]
365         if !exists || len(pkgNames) != 1 {
366                 http.Error(w, "single name is expected in request", http.StatusBadRequest)
367                 return
368         }
369         dir := normalizationRe.ReplaceAllString(pkgNames[0], "-")
370         dirPath := filepath.Join(*root, dir)
371         var digestExpected []byte
372         if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists {
373                 digestExpected, err = hex.DecodeString(digestExpectedHex[0])
374                 if err != nil {
375                         http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest)
376                         return
377                 }
378         }
379         gpgSigsExpected := make(map[string]struct{})
380
381         // Checking is it internal package
382         if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil {
383                 log.Println(r.RemoteAddr, "non-internal package", dir)
384                 http.Error(w, "unknown internal package", http.StatusUnauthorized)
385                 return
386         }
387
388         for _, file := range r.MultipartForm.File["content"] {
389                 filename := file.Filename
390                 gpgSigsExpected[filename+GPGSigExt] = struct{}{}
391                 log.Println(r.RemoteAddr, "put", filename, "by", username)
392                 path := filepath.Join(dirPath, filename)
393                 if _, err = os.Stat(path); err == nil {
394                         log.Println(r.RemoteAddr, "already exists", filename)
395                         http.Error(w, "already exists", http.StatusBadRequest)
396                         return
397                 }
398                 if !mkdirForPkg(w, r, dir) {
399                         return
400                 }
401                 src, err := file.Open()
402                 defer src.Close()
403                 if err != nil {
404                         http.Error(w, err.Error(), http.StatusInternalServerError)
405                         return
406                 }
407                 dst, err := TempFile(dirPath)
408                 if err != nil {
409                         http.Error(w, err.Error(), http.StatusInternalServerError)
410                         return
411                 }
412                 hasher := sha256.New()
413                 wr := io.MultiWriter(hasher, dst)
414                 if _, err = io.Copy(wr, src); err != nil {
415                         os.Remove(dst.Name())
416                         dst.Close()
417                         http.Error(w, err.Error(), http.StatusInternalServerError)
418                         return
419                 }
420                 if err = dst.Sync(); err != nil {
421                         os.Remove(dst.Name())
422                         dst.Close()
423                         http.Error(w, err.Error(), http.StatusInternalServerError)
424                         return
425                 }
426                 dst.Close()
427                 digest := hasher.Sum(nil)
428                 if digestExpected != nil {
429                         if bytes.Compare(digestExpected, digest) == 0 {
430                                 log.Println(r.RemoteAddr, filename, "good checksum received")
431                         } else {
432                                 log.Println(r.RemoteAddr, filename, "bad checksum received")
433                                 http.Error(w, "bad checksum", http.StatusBadRequest)
434                                 os.Remove(dst.Name())
435                                 return
436                         }
437                 }
438                 if err = os.Rename(dst.Name(), path); err != nil {
439                         http.Error(w, err.Error(), http.StatusInternalServerError)
440                         return
441                 }
442                 if err = DirSync(dirPath); err != nil {
443                         http.Error(w, err.Error(), http.StatusInternalServerError)
444                         return
445                 }
446                 if err = WriteFileSync(dirPath, path+SHA256Ext, digest); err != nil {
447                         http.Error(w, err.Error(), http.StatusInternalServerError)
448                         return
449                 }
450         }
451         for _, file := range r.MultipartForm.File["gpg_signature"] {
452                 filename := file.Filename
453                 if _, exists := gpgSigsExpected[filename]; !exists {
454                         http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest)
455                         return
456                 }
457                 delete(gpgSigsExpected, filename)
458                 log.Println(r.RemoteAddr, "put", filename, "by", username)
459                 path := filepath.Join(dirPath, filename)
460                 if _, err = os.Stat(path); err == nil {
461                         log.Println(r.RemoteAddr, "already exists", filename)
462                         http.Error(w, "already exists", http.StatusBadRequest)
463                         return
464                 }
465                 src, err := file.Open()
466                 if err != nil {
467                         http.Error(w, err.Error(), http.StatusInternalServerError)
468                         return
469                 }
470                 sig, err := ioutil.ReadAll(src)
471                 src.Close()
472                 if err != nil {
473                         http.Error(w, err.Error(), http.StatusInternalServerError)
474                         return
475                 }
476                 if err = WriteFileSync(dirPath, path, sig); err != nil {
477                         http.Error(w, err.Error(), http.StatusInternalServerError)
478                         return
479                 }
480         }
481 }
482
483 func handler(w http.ResponseWriter, r *http.Request) {
484         switch r.Method {
485         case "GET":
486                 var path string
487                 var autorefresh bool
488                 var gpgUpdate bool
489                 if strings.HasPrefix(r.URL.Path, *norefreshURLPath) {
490                         path = strings.TrimPrefix(r.URL.Path, *norefreshURLPath)
491                 } else if strings.HasPrefix(r.URL.Path, *refreshURLPath) {
492                         path = strings.TrimPrefix(r.URL.Path, *refreshURLPath)
493                         autorefresh = true
494                 } else if strings.HasPrefix(r.URL.Path, *gpgUpdateURLPath) {
495                         path = strings.TrimPrefix(r.URL.Path, *gpgUpdateURLPath)
496                         autorefresh = true
497                         gpgUpdate = true
498                 } else {
499                         http.Error(w, "unknown action", http.StatusBadRequest)
500                         return
501                 }
502                 parts := strings.Split(strings.TrimSuffix(path, "/"), "/")
503                 if len(parts) > 2 {
504                         http.Error(w, "invalid path", http.StatusBadRequest)
505                         return
506                 }
507                 if len(parts) == 1 {
508                         if parts[0] == "" {
509                                 listRoot(w, r)
510                         } else {
511                                 listDir(w, r, parts[0], autorefresh, gpgUpdate)
512                         }
513                 } else {
514                         servePkg(w, r, parts[0], parts[1])
515                 }
516         case "POST":
517                 serveUpload(w, r)
518         default:
519                 http.Error(w, "unknown action", http.StatusBadRequest)
520         }
521 }
522
523 func goodIntegrity() bool {
524         dirs, err := ioutil.ReadDir(*root)
525         if err != nil {
526                 log.Fatal(err)
527         }
528         hasher := sha256.New()
529         digest := make([]byte, sha256.Size)
530         isGood := true
531         var data []byte
532         var pkgName string
533         for _, dir := range dirs {
534                 files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name()))
535                 if err != nil {
536                         log.Fatal(err)
537                 }
538                 for _, file := range files {
539                         if !strings.HasSuffix(file.Name(), SHA256Ext) {
540                                 continue
541                         }
542                         pkgName = strings.TrimSuffix(file.Name(), SHA256Ext)
543                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName))
544                         if err != nil {
545                                 if os.IsNotExist(err) {
546                                         continue
547                                 }
548                                 log.Fatal(err)
549                         }
550                         hasher.Write(data)
551                         data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name()))
552                         if err != nil {
553                                 log.Fatal(err)
554                         }
555                         if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 {
556                                 log.Println(pkgName, "GOOD")
557                         } else {
558                                 isGood = false
559                                 log.Println(pkgName, "BAD")
560                         }
561                         hasher.Reset()
562                 }
563         }
564         return isGood
565 }
566
567 func main() {
568         flag.Parse()
569         if *warranty {
570                 fmt.Println(Warranty)
571                 return
572         }
573         if *version {
574                 fmt.Println("GoCheese version " + Version + " built with " + runtime.Version())
575                 return
576         }
577         if *fsck {
578                 if !goodIntegrity() {
579                         os.Exit(1)
580                 }
581                 return
582         }
583         if *passwdCheck {
584                 refreshPasswd()
585                 return
586         }
587         if (*tlsCert != "" && *tlsKey == "") || (*tlsCert == "" && *tlsKey != "") {
588                 log.Fatalln("Both -tls-cert and -tls-key are required")
589         }
590         refreshPasswd()
591         log.Println("root:", *root, "bind:", *bind)
592
593         ln, err := net.Listen("tcp", *bind)
594         if err != nil {
595                 log.Fatal(err)
596         }
597         ln = netutil.LimitListener(ln, *maxClients)
598         server := &http.Server{
599                 ReadTimeout:  time.Minute,
600                 WriteTimeout: time.Minute,
601         }
602         http.HandleFunc(*norefreshURLPath, handler)
603         http.HandleFunc(*refreshURLPath, handler)
604         http.HandleFunc(*gpgUpdateURLPath, handler)
605
606         needsRefreshPasswd := make(chan os.Signal, 0)
607         needsShutdown := make(chan os.Signal, 0)
608         exitErr := make(chan error, 0)
609         signal.Notify(needsRefreshPasswd, syscall.SIGHUP)
610         signal.Notify(needsShutdown, syscall.SIGTERM, syscall.SIGINT)
611         go func() {
612                 for range needsRefreshPasswd {
613                         log.Println("Refreshing passwords")
614                         refreshPasswd()
615                 }
616         }()
617         go func(s *http.Server) {
618                 <-needsShutdown
619                 killed = true
620                 log.Println("Shutting down")
621                 ctx, cancel := context.WithTimeout(context.TODO(), time.Minute)
622                 exitErr <- s.Shutdown(ctx)
623                 cancel()
624         }(server)
625
626         if *tlsCert == "" {
627                 err = server.Serve(ln)
628         } else {
629                 err = server.ServeTLS(ln, *tlsCert, *tlsKey)
630         }
631         if err != http.ErrServerClosed {
632                 log.Fatal(err)
633         }
634         if err := <-exitErr; err != nil {
635                 log.Fatal(err)
636         }
637 }