From 67e9a27b34099925ab03353de7b990688387c925 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 18 Nov 2014 02:09:50 -0800 Subject: [PATCH] Enable way to split files at input bytes --- Makefile | 5 +- pkgs/split/.gitignore | 1 + pkgs/split/Makefile | 11 ++++ pkgs/split/TESTFILE | Bin 0 -> 262144 bytes pkgs/split/split.c | 120 +++++++++++++++++++++++++++++++++++++++ pkgs/split/split.go | 53 +++++++++++++++++ pkgs/split/split.h | 22 +++++++ pkgs/split/split_test.go | 34 +++++++++++ 8 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 pkgs/split/.gitignore create mode 100644 pkgs/split/Makefile create mode 100644 pkgs/split/TESTFILE create mode 100644 pkgs/split/split.c create mode 100644 pkgs/split/split.go create mode 100644 pkgs/split/split.h create mode 100644 pkgs/split/split_test.go diff --git a/Makefile b/Makefile index 748535dbf..c3a0170e6 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,10 @@ build-erasure: build-signify: @cd pkgs/signify && ${MAKE} ${MAKE_OPTIONS} -cover: build-erasure build-signify +build-split: + @cd pkgs/split && ${MAKE} ${MAKE_OPTIONS} + +cover: build-erasure build-signify build-split @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/storage @godep go test -race -coverprofile=cover.out github.com/minio-io/minio/pkgs/gateway diff --git a/pkgs/split/.gitignore b/pkgs/split/.gitignore new file mode 100644 index 000000000..411512991 --- /dev/null +++ b/pkgs/split/.gitignore @@ -0,0 +1 @@ +TESTFILE.* \ No newline at end of file diff --git a/pkgs/split/Makefile b/pkgs/split/Makefile new file mode 100644 index 000000000..5e12bca9b --- /dev/null +++ b/pkgs/split/Makefile @@ -0,0 +1,11 @@ +all: build test +.PHONY: all + +build: + @godep go build + +test: build + @godep go test -race -coverprofile=cover.out + +clean: + @rm -v TESTFILE.* cover.out diff --git a/pkgs/split/TESTFILE b/pkgs/split/TESTFILE new file mode 100644 index 0000000000000000000000000000000000000000..6d23118f0d0084657a974875123ddc1b9a0738dd GIT binary patch literal 262144 zcmeIuF#!Mo0K%a4Pwj07h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA Wz<>b*1`HT5V8DO@0|pEj_<;ccJOBXz literal 0 HcmV?d00001 diff --git a/pkgs/split/split.c b/pkgs/split/split.c new file mode 100644 index 000000000..0a3cef41a --- /dev/null +++ b/pkgs/split/split.c @@ -0,0 +1,120 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "split.h" + +static +size_t _get_filesize(int fd) +{ + struct stat st; + assert(fstat(fd, &st) != -1); + return (ssize_t) st.st_size; +} + +static +ssize_t read_write_chunk(int in, int out, ssize_t bytes) +{ + ssize_t n, m; + char *buf = NULL; + + buf = calloc(bytes, 1); + assert(buf != NULL); + + n = read(in, buf, bytes); + if (n < 0) + return -1; + + m = write(out, buf, n); + if (m < 0) + return -1; + + if (buf) + free(buf); + return m; +} + +static int +_allocate_newchunk(char *chunkname) +{ + + int chunk = -1; + + if (!chunkname) + return -1; + + if ((strlen(chunkname)) >= MAXPATHLEN) { + fprintf (stderr, "chunkname + suffix too long"); + return -1; + } + + chunk = open (chunkname, O_CREAT|O_WRONLY, S_IRUSR|S_IWUSR); + return chunk; +} + +/* + * Generate chunks by chunking at input bytes. + */ +int +minio_split(char *filename, ssize_t bytecnt) +{ + ssize_t fsize; + int output = -1; + int input = -1; + int remainder = 0; + char newchunk[MAXPATHLEN] = {0,}; + int chunk_size = 0; + int i = 0; + + if (bytecnt < 0) + return -1; + + if ((input = open(filename, O_RDONLY)) < 0) + return -1; + + fsize = _get_filesize(input); + remainder = fsize % bytecnt; + + if (remainder == 0) + chunk_size = fsize / bytecnt; + else + chunk_size = (fsize + (bytecnt - remainder)) / bytecnt; + + if (chunk_size == 0) + return -1; + + for (i = 0; i < chunk_size; i++) { + snprintf (newchunk, sizeof(newchunk)-1, "%s.%d", filename, i); + + if ((output = _allocate_newchunk(newchunk)) < 0) + return -1; + + if (read_write_chunk(input, output, bytecnt) < 0) + return -1; + } + return 0; +} diff --git a/pkgs/split/split.go b/pkgs/split/split.go new file mode 100644 index 000000000..64dc7cae2 --- /dev/null +++ b/pkgs/split/split.go @@ -0,0 +1,53 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// +build linux +// amd64 + +package split + +// #include +// #include +// +// #include "split.h" +import "C" +import ( + "errors" + "github.com/minio-io/minio/pkgs/strbyteconv" + "unsafe" +) + +type Split struct { + bytecnt C.ssize_t + bname *C.char +} + +func (b *Split) GenChunks(bname string, bytestr string) error { + bytecnt, err := strbyteconv.StringToBytes(bytestr) + if err != nil { + return err + } + + b.bytecnt = C.ssize_t(bytecnt) + b.bname = C.CString(bname) + defer C.free(unsafe.Pointer(b.bname)) + + value := C.minio_split(b.bname, b.bytecnt) + if value < 0 { + return errors.New("File split failed") + } + return nil +} diff --git a/pkgs/split/split.h b/pkgs/split/split.h new file mode 100644 index 000000000..f8ccca479 --- /dev/null +++ b/pkgs/split/split.h @@ -0,0 +1,22 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __SPLIT_H__ +#define __SPLIT_H__ + +int minio_split(char *filename, ssize_t bytecnt); + +#endif /* __SPLIT_H__ */ diff --git a/pkgs/split/split_test.go b/pkgs/split/split_test.go new file mode 100644 index 000000000..62dc28ebd --- /dev/null +++ b/pkgs/split/split_test.go @@ -0,0 +1,34 @@ +/* + * Mini Object Storage, (C) 2014 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package split + +import ( + . "gopkg.in/check.v1" + "testing" +) + +type MySuite struct{} + +var _ = Suite(&MySuite{}) + +func Test(t *testing.T) { TestingT(t) } + +func (s *MySuite) TestFileSplit(c *C) { + b := Split{} + err := b.GenChunks("TESTFILE", "20KB") + c.Assert(err, IsNil) +}