shithub: s3

Info  •  Files  •  Log  •  Branches

write & cp: work for large files

--- a/cmd.c

+++ b/cmd.c

@@ -21,7 +21,7 @@

 	}

 }

-static void

+void

 dumperr(Hcon *con)

 {

 	long n;

--- a/cmd.h

+++ b/cmd.h

@@ -2,3 +2,4 @@

 void downloadrange(S3*, char*, Biobuf*, long, long);

 int parseuri(S3*, char*, int, char*);

 int parseargs(S3*, int, char**);

+void dumperr(Hcon*);

--- a/cp.c

+++ b/cp.c

@@ -37,7 +37,7 @@

 upload(S3 *s3, char *localpath, char *remotepath)

 {

 	DigestState *ds;

-	uchar data[8192];

+	uchar data[1 * 1024 * 1024];

 	char mime[32];

 	uchar digest[SHA2_256dlen];

 	long n;

@@ -44,11 +44,13 @@

 	int fd;

 	char buf[256];

 	Hcon con;

+	long length;

 	mimetype(localpath, mime, sizeof mime);

 	fd = open(localpath, OREAD);

 	if(fd < 0)

 		sysfatal("upload open: %r");

+	length = 0;

 	for(ds = nil;;){

 		n = read(fd, data, sizeof data);

 		if(n < 0)

@@ -55,12 +57,13 @@

 			sysfatal("file read: %r");

 		if(n == 0)

 			break;

+		length += n;

 		ds = sha2_256(data, n, nil, ds);

 	}

 	sha2_256(nil, 0, digest, ds);

 	seek(fd, 0, 0);

-	if(s3put(s3, &con, remotepath, mime, digest) < 0)

+	if(s3put(s3, &con, remotepath, mime, digest, length) < 0)

 		sysfatal("upload postbody open: %r");

 	for(;;){

 		n = read(fd, buf, sizeof buf);

@@ -71,8 +74,10 @@

 		if(write(con.post, buf, n) < 0)

 			sysfatal("upload write: %r");

 	}

-	if(hdone(&con) < 0)

+	if(hdone(&con) < 0){

+		dumperr(&con);

 		sysfatal("error response code: %r");

+	}

 	hclose(&con);

 }

--- a/s3.c

+++ b/s3.c

@@ -49,6 +49,7 @@

 	char time[128];

 	char authhdr[512];

 	char *range;

+	long length;

 } Hreq;

 static void

@@ -96,6 +97,7 @@

 	hreq->payhash = payhash;

 	hreq->mime = mime;

 	hreq->range = nil;

+	hreq->length = -1;

 }

 static void

@@ -171,6 +173,8 @@

 		return -1;

 	if(hreq->range != nil && ctlprint(cfd, "headers range: %s", hreq->range) < 0)

 		return -1;

+	if(hreq->length != -1 && ctlprint(cfd, "headers content-length: %ld", hreq->length) < 0)

+		return -1;

 	return 0;

 }

@@ -279,11 +283,12 @@

 }

 int

-s3put(S3 *s3, Hcon *con, char *path, char *mime, uchar *payhash)

+s3put(S3 *s3, Hcon *con, char *path, char *mime, uchar *payhash, long length)

 {

 	Hreq h;

 	mkreq(&h, "PUT", path, payhash, mime);

+	h.length = length;

 	return hopen(con, s3, ORDWR, &h);

 }

--- a/s3.h

+++ b/s3.h

@@ -17,7 +17,7 @@

 int s3get(S3 *s3, Hcon *con, char *path);

 int s3del(S3 *s3, Hcon *con, char *path);

-int s3put(S3 *s3, Hcon *con, char *path, char *mime, uchar *payhash);

+int s3put(S3 *s3, Hcon *con, char *path, char *mime, uchar *payhash, long length);

 int s3post(S3 *s3, Hcon *con, char *path);

 int s3postwrite(S3 *s3, Hcon *con, char *path, char *mime, uchar *payhash);

 int s3getrange(S3 *s3, Hcon *con, char *path, long off, long n);

--- a/write.c

+++ b/write.c

@@ -24,7 +24,7 @@

 	long etagn;

 	sha2_256(data, n, payhash, nil);

-	ret = s3put(s3, &con, smprint("%s?partNumber=%d&uploadId=%s", path, partno, uploadid), "application/octet-stream", payhash);

+	ret = s3put(s3, &con, smprint("%s?partNumber=%d&uploadId=%s", path, partno, uploadid), "application/octet-stream", payhash, n);

 	if(ret < 0)

 		sysfatal("part %d upload: s3put: %r", partno);

 	if(write(con.post, data, n) != n)

-- 

⑨