X-Git-Url: http://www.git.cypherpunks.ru/?p=gocheese.git;a=blobdiff_plain;f=gocheese.go;h=c1f13f6b193a4ddfe185ef95a3bb7636836fddda;hp=40f172bc9eedb50d5c337572d5da5036f760ac15;hb=b036ee436eb9bd8889734232a22d3f24be5c9ee2;hpb=984e32455cd547f697b93fea7d88b5adb160513b diff --git a/gocheese.go b/gocheese.go index 40f172b..c1f13f6 100644 --- a/gocheese.go +++ b/gocheese.go @@ -20,17 +20,11 @@ along with this program. If not, see . package main import ( - "bufio" "bytes" "context" - "crypto/md5" - "crypto/sha256" - "crypto/sha512" "encoding/hex" "flag" "fmt" - "hash" - "io" "io/ioutil" "log" "net" @@ -60,7 +54,6 @@ const ( HTMLElement = " %s
\n" InternalFlag = ".internal" GPGSigExt = ".asc" - GPGSigAttr = " data-gpg-sig=true" Warranty = `This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -76,8 +69,19 @@ along with this program. If not, see .` ) var ( - pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) - Version string = "UNKNOWN" + pkgPyPI = regexp.MustCompile(`^.*]*>(.+)
.*$`) + normalizationRe = regexp.MustCompilePOSIX("[-_.]+") + + HashAlgoSHA256 = "sha256" + HashAlgoBLAKE2b256 = "blake2_256" + HashAlgoSHA512 = "sha512" + HashAlgoMD5 = "md5" + knownHashAlgos []string = []string{ + HashAlgoSHA256, + HashAlgoBLAKE2b256, + HashAlgoSHA512, + HashAlgoMD5, + } root = flag.String("root", "./packages", "Path to packages directory") bind = flag.String("bind", "[::]:8080", "Address to bind to") @@ -94,10 +98,9 @@ var ( version = flag.Bool("version", false, "Print version information") warranty = flag.Bool("warranty", false, "Print warranty information") - killed bool - - pypiURLParsed *url.URL - normalizationRe *regexp.Regexp = regexp.MustCompilePOSIX("[-_.]+") + Version string = "UNKNOWN" + killed bool + pypiURLParsed *url.URL ) func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { @@ -112,216 +115,6 @@ func mkdirForPkg(w http.ResponseWriter, r *http.Request, dir string) bool { return true } -func refreshDir( - w http.ResponseWriter, - r *http.Request, - dir, - filenameGet string, - gpgUpdate bool, -) bool { - if _, err := os.Stat(filepath.Join(*root, dir, InternalFlag)); err == nil { - return true - } - resp, err := http.Get(*pypiURL + dir + "/") - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - if !mkdirForPkg(w, r, dir) { - return false - } - dirPath := filepath.Join(*root, dir) - var submatches []string - var uri string - var filename string - var path string - var pkgURL *url.URL - var digest []byte - for _, lineRaw := range bytes.Split(body, []byte("\n")) { - submatches = pkgPyPI.FindStringSubmatch(string(lineRaw)) - if len(submatches) == 0 { - continue - } - uri = submatches[1] - filename = submatches[2] - if pkgURL, err = url.Parse(uri); err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - - if pkgURL.Fragment == "" { - log.Println(r.RemoteAddr, "pypi", filename, "no digest provided") - http.Error(w, "no digest provided", http.StatusBadGateway) - return false - } - digestInfo := strings.Split(pkgURL.Fragment, "=") - if len(digestInfo) == 1 { - // Ancient non PEP-0503 PyPIs, assume MD5 - digestInfo = []string{"md5", digestInfo[0]} - } else if len(digestInfo) != 2 { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest provided") - http.Error(w, "invalid digest provided", http.StatusBadGateway) - return false - } - digest, err = hex.DecodeString(digestInfo[1]) - if err != nil { - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - var hasherNew func() hash.Hash - var hashExt string - var hashSize int - switch digestInfo[0] { - case "md5": - hashExt = ".md5" - hasherNew = md5.New - hashSize = md5.Size - case "sha256": - hashExt = ".sha256" - hasherNew = sha256.New - hashSize = sha256.Size - case "sha512": - hashExt = ".sha512" - hasherNew = sha512.New - hashSize = sha512.Size - default: - log.Println( - r.RemoteAddr, "pypi", filename, - "unknown digest algorithm", digestInfo[0], - ) - http.Error(w, "unknown digest algorithm", http.StatusBadGateway) - return false - } - if len(digest) != hashSize { - log.Println(r.RemoteAddr, "pypi", filename, "invalid digest length") - http.Error(w, "invalid digest length", http.StatusBadGateway) - return false - } - - pkgURL.Fragment = "" - if pkgURL.Host == "" { - uri = pypiURLParsed.ResolveReference(pkgURL).String() - } else { - uri = pkgURL.String() - } - path = filepath.Join(dirPath, filename) - if filename == filenameGet { - if killed { - // Skip heavy remote call, when shutting down - http.Error(w, "shutting down", http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi download", filename) - resp, err = http.Get(uri) - if err != nil { - log.Println(r.RemoteAddr, "pypi download error:", err.Error()) - http.Error(w, err.Error(), http.StatusBadGateway) - return false - } - defer resp.Body.Close() - hasher := hasherNew() - hasherOur := sha256.New() - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - dstBuf := bufio.NewWriter(dst) - wrs := []io.Writer{hasher, dstBuf} - if hashExt != ".sha256" { - wrs = append(wrs, hasherOur) - } - wr := io.MultiWriter(wrs...) - if _, err = io.Copy(wr, resp.Body); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = dstBuf.Flush(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if bytes.Compare(hasher.Sum(nil), digest) != 0 { - log.Println(r.RemoteAddr, "pypi", filename, "digest mismatch") - os.Remove(dst.Name()) - dst.Close() - http.Error(w, "digest mismatch", http.StatusBadGateway) - return false - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = dst.Close(); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - if hashExt != ".sha256" { - hashExt = ".sha256" - digest = hasherOur.Sum(nil) - } - } - if filename == filenameGet || gpgUpdate { - if _, err = os.Stat(path); err != nil { - goto GPGSigSkip - } - resp, err := http.Get(uri + GPGSigExt) - if err != nil { - goto GPGSigSkip - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - goto GPGSigSkip - } - sig, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - goto GPGSigSkip - } - if err = WriteFileSync(dirPath, path+GPGSigExt, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi downloaded signature", filename) - } - GPGSigSkip: - path = path + hashExt - _, err = os.Stat(path) - if err == nil { - continue - } - if !os.IsNotExist(err) { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - log.Println(r.RemoteAddr, "pypi touch", filename) - if err = WriteFileSync(dirPath, path, digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return false - } - } - return true -} - func listRoot(w http.ResponseWriter, r *http.Request) { files, err := ioutil.ReadDir(*root) if err != nil { @@ -363,54 +156,44 @@ func listDir( http.Error(w, err.Error(), http.StatusInternalServerError) return } - var result bytes.Buffer - result.WriteString(fmt.Sprintf(HTMLBegin, dir)) - var digest []byte - var gpgSigAttr string - var fnClean string files := make(map[string]struct{}, len(fis)/2) for _, fi := range fis { files[fi.Name()] = struct{}{} } - for _, algoExt := range []string{".sha256", ".sha512", ".md5"} { + var result bytes.Buffer + result.WriteString(fmt.Sprintf(HTMLBegin, dir)) + for _, algo := range knownHashAlgos { for fn, _ := range files { if killed { // Skip expensive I/O when shutting down http.Error(w, "shutting down", http.StatusInternalServerError) return } - if !strings.HasSuffix(fn, algoExt) { + if !strings.HasSuffix(fn, "."+algo) { continue } - digest, err = ioutil.ReadFile(filepath.Join(dirPath, fn)) + delete(files, fn) + digest, err := ioutil.ReadFile(filepath.Join(dirPath, fn)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } - fnClean = strings.TrimSuffix(fn, algoExt) - if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); os.IsNotExist(err) { - gpgSigAttr = "" - } else { - gpgSigAttr = GPGSigAttr + fnClean := strings.TrimSuffix(fn, "."+algo) + delete(files, fnClean) + gpgSigAttr := "" + if _, err = os.Stat(filepath.Join(dirPath, fnClean+GPGSigExt)); err == nil { + gpgSigAttr = " data-gpg-sig=true" + delete(files, fnClean+GPGSigExt) } result.WriteString(fmt.Sprintf( HTMLElement, strings.Join([]string{ *refreshURLPath, dir, "/", fnClean, - "#", algoExt[1:], "=", hex.EncodeToString(digest), + "#", algo, "=", hex.EncodeToString(digest), }, ""), gpgSigAttr, fnClean, )) - for _, n := range []string{ - fnClean, - fnClean + GPGSigExt, - fnClean + ".sha256", - fnClean + ".sha512", - fnClean + ".md5", - } { - delete(files, n) - } } } result.WriteString(HTMLEnd) @@ -428,153 +211,6 @@ func servePkg(w http.ResponseWriter, r *http.Request, dir, filename string) { http.ServeFile(w, r, path) } -func serveUpload(w http.ResponseWriter, r *http.Request) { - // Authentication - username, password, ok := r.BasicAuth() - if !ok { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - auther, ok := passwords[username] - if !ok || !auther.Auth(password) { - log.Println(r.RemoteAddr, "unauthenticated", username) - http.Error(w, "unauthenticated", http.StatusUnauthorized) - return - } - - // Form parsing - var err error - if err = r.ParseMultipartForm(1 << 20); err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - pkgNames, exists := r.MultipartForm.Value["name"] - if !exists || len(pkgNames) != 1 { - http.Error(w, "single name is expected in request", http.StatusBadRequest) - return - } - dir := normalizationRe.ReplaceAllString(pkgNames[0], "-") - dirPath := filepath.Join(*root, dir) - var digestExpected []byte - if digestExpectedHex, exists := r.MultipartForm.Value["sha256_digest"]; exists { - digestExpected, err = hex.DecodeString(digestExpectedHex[0]) - if err != nil { - http.Error(w, "bad sha256_digest: "+err.Error(), http.StatusBadRequest) - return - } - } - gpgSigsExpected := make(map[string]struct{}) - - // Checking is it internal package - if _, err = os.Stat(filepath.Join(dirPath, InternalFlag)); err != nil { - log.Println(r.RemoteAddr, "non-internal package", dir) - http.Error(w, "unknown internal package", http.StatusUnauthorized) - return - } - - for _, file := range r.MultipartForm.File["content"] { - filename := file.Filename - gpgSigsExpected[filename+GPGSigExt] = struct{}{} - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - if !mkdirForPkg(w, r, dir) { - return - } - src, err := file.Open() - defer src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst, err := TempFile(dirPath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dstBuf := bufio.NewWriter(dst) - hasher := sha256.New() - wr := io.MultiWriter(hasher, dst) - if _, err = io.Copy(wr, src); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dstBuf.Flush(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = dst.Sync(); err != nil { - os.Remove(dst.Name()) - dst.Close() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - dst.Close() - digest := hasher.Sum(nil) - if digestExpected != nil { - if bytes.Compare(digestExpected, digest) == 0 { - log.Println(r.RemoteAddr, filename, "good checksum received") - } else { - log.Println(r.RemoteAddr, filename, "bad checksum received") - http.Error(w, "bad checksum", http.StatusBadRequest) - os.Remove(dst.Name()) - return - } - } - if err = os.Rename(dst.Name(), path); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = DirSync(dirPath); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path+".sha256", digest); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } - for _, file := range r.MultipartForm.File["gpg_signature"] { - filename := file.Filename - if _, exists := gpgSigsExpected[filename]; !exists { - http.Error(w, "unexpected GPG signature filename", http.StatusBadRequest) - return - } - delete(gpgSigsExpected, filename) - log.Println(r.RemoteAddr, "put", filename, "by", username) - path := filepath.Join(dirPath, filename) - if _, err = os.Stat(path); err == nil { - log.Println(r.RemoteAddr, "already exists", filename) - http.Error(w, "already exists", http.StatusBadRequest) - return - } - src, err := file.Open() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - sig, err := ioutil.ReadAll(src) - src.Close() - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if err = WriteFileSync(dirPath, path, sig); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - } -} - func handler(w http.ResponseWriter, r *http.Request) { switch r.Method { case "GET": @@ -615,50 +251,6 @@ func handler(w http.ResponseWriter, r *http.Request) { } } -func goodIntegrity() bool { - dirs, err := ioutil.ReadDir(*root) - if err != nil { - log.Fatal(err) - } - hasher := sha256.New() - digest := make([]byte, sha256.Size) - isGood := true - var data []byte - var pkgName string - for _, dir := range dirs { - files, err := ioutil.ReadDir(filepath.Join(*root, dir.Name())) - if err != nil { - log.Fatal(err) - } - for _, file := range files { - if !strings.HasSuffix(file.Name(), ".sha256") { - continue - } - pkgName = strings.TrimSuffix(file.Name(), ".sha256") - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), pkgName)) - if err != nil { - if os.IsNotExist(err) { - continue - } - log.Fatal(err) - } - hasher.Write(data) - data, err = ioutil.ReadFile(filepath.Join(*root, dir.Name(), file.Name())) - if err != nil { - log.Fatal(err) - } - if bytes.Compare(hasher.Sum(digest[:0]), data) == 0 { - fmt.Println(pkgName, "GOOD") - } else { - isGood = false - fmt.Println(pkgName, "BAD") - } - hasher.Reset() - } - } - return isGood -} - func main() { flag.Parse() if *warranty {