Add GetObjectNInfo to object layer (#6449)

The new call combines GetObjectInfo and GetObject, and returns an
object with a ReadCloser interface.

Also adds a number of end-to-end encryption tests at the handler
level.

cmd/encryption-v1.go

@@ -82,7 +82,7 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
 
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
 // It returns the client provided key on success.
-func ParseSSECopyCustomerRequest(r *http.Request, metadata map[string]string) (key []byte, err error) {
+func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
 	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
 		// we cannot use r.TLS == nil here because Go's http implementation reflects on
 		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
@@ -90,10 +90,10 @@ func ParseSSECopyCustomerRequest(r *http.Request, metadata map[string]string) (k
 		// will always fail -> r.TLS is always nil even for TLS requests.
 		return nil, errInsecureSSERequest
 	}
-	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(r.Header) {
+	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
 		return nil, crypto.ErrIncompatibleEncryptionMethod
 	}
-	k, err := crypto.SSECopy.ParseHTTP(r.Header)
+	k, err := crypto.SSECopy.ParseHTTP(h)
 	return k[:], err
 }
 
@@ -240,7 +240,7 @@ func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string
 		err error
 	)
 	if crypto.SSECopy.IsRequested(r.Header) {
-		key, err = ParseSSECopyCustomerRequest(r, metadata)
+		key, err = ParseSSECopyCustomerRequest(r.Header, metadata)
 		if err != nil {
 			return nil, err
 		}
@@ -312,6 +312,127 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,
 	return writer, nil
 }
 
+// Adding support for reader based interface
+
+// DecryptRequestWithSequenceNumberR - same as
+// DecryptRequestWithSequenceNumber but with a reader
+func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	if crypto.S3.IsEncrypted(metadata) {
+		return newDecryptReader(client, nil, bucket, object, seqNumber, metadata)
+	}
+
+	key, err := ParseSSECustomerHeader(h)
+	if err != nil {
+		return nil, err
+	}
+	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
+}
+
+// DecryptCopyRequestR - same as DecryptCopyRequest, but with a
+// Reader
+func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, metadata map[string]string) (io.Reader, error) {
+	var (
+		key []byte
+		err error
+	)
+	if crypto.SSECopy.IsRequested(h) {
+		key, err = ParseSSECopyCustomerRequest(h, metadata)
+		if err != nil {
+			return nil, err
+		}
+	}
+	delete(metadata, crypto.SSECopyKey) // make sure we do not save the key by accident
+	return newDecryptReader(client, key, bucket, object, 0, metadata)
+}
+
+func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
+	if err != nil {
+		return nil, err
+	}
+	return newDecryptReaderWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
+}
+
+func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	reader, err := sio.DecryptReader(client, sio.Config{
+		Key:            objectEncryptionKey,
+		SequenceNumber: seqNumber,
+	})
+	if err != nil {
+		return nil, crypto.ErrInvalidCustomerKey
+	}
+	return reader, nil
+}
+
+// GetEncryptedOffsetLength - returns encrypted offset and length
+// along with sequence number
+func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
+	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
+		return
+	}
+	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
+	return
+}
+
+// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
+// reader
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
+	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
+	io.Reader, error) {
+
+	bucket, object := oi.Bucket, oi.Name
+
+	// Single part case
+	if len(oi.Parts) == 0 || !crypto.IsMultiPart(oi.UserDefined) {
+		var reader io.Reader
+		var err error
+		if copySource {
+			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, oi.UserDefined)
+		} else {
+			reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
+		}
+		if err != nil {
+			return nil, err
+		}
+		return reader, nil
+	}
+
+	partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
+	partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+
+	w := &DecryptBlocksReader{
+		reader:            inputReader,
+		startSeqNum:       seqNumber,
+		partDecRelOffset:  partDecRelOffset,
+		partEncRelOffset:  partEncRelOffset,
+		parts:             oi.Parts,
+		partIndex:         partStart,
+		header:            h,
+		bucket:            bucket,
+		object:            object,
+		customerKeyHeader: h.Get(crypto.SSECKey),
+		copySource:        copySource,
+	}
+
+	w.metadata = map[string]string{}
+	// Copy encryption metadata for internal use.
+	for k, v := range oi.UserDefined {
+		w.metadata[k] = v
+	}
+
+	if w.copySource {
+		w.customerKeyHeader = h.Get(crypto.SSECopyKey)
+	}
+
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
+		return nil, err
+	}
+
+	return w, nil
+}
+
 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
 func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
@@ -333,7 +454,131 @@ func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, me
 	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }
 
-// DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
+// DecryptBlocksReader - decrypts multipart parts, while implementing
+// a io.Reader compatible interface.
+type DecryptBlocksReader struct {
+	// Source of the encrypted content that will be decrypted
+	reader io.Reader
+	// Current decrypter for the current encrypted data block
+	decrypter io.Reader
+	// Start sequence number
+	startSeqNum uint32
+	// Current part index
+	partIndex int
+	// Parts information
+	parts          []objectPartInfo
+	header         http.Header
+	bucket, object string
+	metadata       map[string]string
+
+	partDecRelOffset, partEncRelOffset int64
+
+	copySource bool
+	// Customer Key
+	customerKeyHeader string
+}
+
+func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
+	m := make(map[string]string)
+	for k, v := range d.metadata {
+		m[k] = v
+	}
+	// Initialize the first decrypter; new decrypters will be
+	// initialized in Read() operation as needed.
+	var key []byte
+	var err error
+	if d.copySource {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
+			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
+		}
+	} else {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECKey, d.customerKeyHeader)
+			key, err = ParseSSECustomerHeader(d.header)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	objectEncryptionKey, err := decryptObjectInfo(key, d.bucket, d.object, m)
+	if err != nil {
+		return err
+	}
+
+	var partIDbin [4]byte
+	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
+
+	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+	mac.Write(partIDbin[:])
+	partEncryptionKey := mac.Sum(nil)
+
+	// make sure we do not save the key by accident
+	if d.copySource {
+		delete(m, crypto.SSECopyKey)
+	} else {
+		delete(m, crypto.SSECKey)
+	}
+
+	// Limit the reader, so the decryptor doesnt receive bytes
+	// from the next part (different DARE stream)
+	encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset
+	decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum, m)
+	if err != nil {
+		return err
+	}
+
+	d.decrypter = decrypter
+	return nil
+}
+
+func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
+	var err error
+	var n1 int
+	decPartSize, _ := sio.DecryptedSize(uint64(d.parts[d.partIndex].Size))
+	unreadPartLen := int64(decPartSize) - d.partDecRelOffset
+	if int64(len(p)) < unreadPartLen {
+		n1, err = d.decrypter.Read(p)
+		if err != nil {
+			return 0, err
+		}
+		d.partDecRelOffset += int64(n1)
+	} else {
+		n1, err = io.ReadFull(d.decrypter, p[:unreadPartLen])
+		if err != nil {
+			return 0, err
+		}
+
+		// We should now proceed to next part, reset all
+		// values appropriately.
+		d.partEncRelOffset = 0
+		d.partDecRelOffset = 0
+		d.startSeqNum = 0
+
+		d.partIndex++
+		if d.partIndex == len(d.parts) {
+			return n1, io.EOF
+		}
+
+		err = d.buildDecrypter(d.parts[d.partIndex].Number)
+		if err != nil {
+			return 0, err
+		}
+
+		n1, err = d.decrypter.Read(p[n1:])
+		if err != nil {
+			return 0, err
+		}
+
+		d.partDecRelOffset += int64(n1)
+	}
+
+	return len(p), nil
+}
+
+// DecryptBlocksWriter - decrypts  multipart parts, while implementing
+// a io.Writer compatible interface.
 type DecryptBlocksWriter struct {
 	// Original writer where the plain data will be written
 	writer io.Writer
@@ -367,7 +612,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 	if w.copySource {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
 			w.req.Header.Set(crypto.SSECopyKey, w.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(w.req, w.metadata)
+			key, err = ParseSSECopyCustomerRequest(w.req.Header, w.metadata)
 		}
 	} else {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
@@ -666,6 +911,119 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	return size, nil
 }
 
+// GetDecryptedRange - To decrypt the range (off, length) of the
+// decrypted object stream, we need to read the range (encOff,
+// encLength) of the encrypted object stream to decrypt it, and
+// compute skipLen, the number of bytes to skip in the beginning of
+// the encrypted range.
+//
+// In addition we also compute the object part number for where the
+// requested range starts, along with the DARE sequence number within
+// that part. For single part objects, the partStart will be 0.
+func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) {
+	if !crypto.IsEncrypted(o.UserDefined) {
+		err = errors.New("Object is not encrypted")
+		return
+	}
+
+	if rs == nil {
+		// No range, so offsets refer to the whole object.
+		return 0, int64(o.Size), 0, 0, 0, nil
+	}
+
+	// Assemble slice of (decrypted) part sizes in `sizes`
+	var decObjSize int64 // decrypted total object size
+	var partSize uint64
+	partSize, err = sio.DecryptedSize(uint64(o.Size))
+	if err != nil {
+		return
+	}
+	sizes := []int64{int64(partSize)}
+	decObjSize = sizes[0]
+	if crypto.IsMultiPart(o.UserDefined) {
+		sizes = make([]int64, len(o.Parts))
+		decObjSize = 0
+		for i, part := range o.Parts {
+			partSize, err = sio.DecryptedSize(uint64(part.Size))
+			if err != nil {
+				return
+			}
+			t := int64(partSize)
+			sizes[i] = t
+			decObjSize += t
+		}
+	}
+
+	var off, length int64
+	off, length, err = rs.GetOffsetLength(decObjSize)
+	if err != nil {
+		return
+	}
+
+	// At this point, we have:
+	//
+	// 1. the decrypted part sizes in `sizes` (single element for
+	//    single part object) and total decrypted object size `decObjSize`
+	//
+	// 2. the (decrypted) start offset `off` and (decrypted)
+	//    length to read `length`
+	//
+	// These are the inputs to the rest of the algorithm below.
+
+	// Locate the part containing the start of the required range
+	var partEnd int
+	var cumulativeSum, encCumulativeSum int64
+	for i, size := range sizes {
+		if off < cumulativeSum+size {
+			partStart = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partStart is always found in the loop above,
+	// because off is validated.
+
+	sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
+
+	// Now we can calculate the number of bytes to skip
+	skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
+
+	encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
+	// Locate the part containing the end of the required range
+	endOffset := off + length - 1
+	for i1, size := range sizes[partStart:] {
+		i := partStart + i1
+		if endOffset < cumulativeSum+size {
+			partEnd = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partEnd is always found in the loop above, because off and
+	// length are validated.
+	endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
+	// Compute endEncOffset with one additional DARE package (so
+	// we read the package containing the last desired byte).
+	endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
+	// Check if the DARE package containing the end offset is a
+	// full sized package (as the last package in the part may be
+	// smaller)
+	lastPartSize, _ := sio.EncryptedSize(uint64(sizes[partEnd]))
+	if endEncOffset > encCumulativeSum+int64(lastPartSize) {
+		endEncOffset = encCumulativeSum + int64(lastPartSize)
+	}
+	encLength = endEncOffset - encOff
+	// Set the sequence number as the starting package number of
+	// the requested block
+	seqNumber = uint32(startPkgNum)
+	return encOff, encLength, skipLen, seqNumber, partStart, nil
+}
+
 // EncryptedSize returns the size of the object after encryption.
 // An encrypted object is always larger than a plain object
 // except for zero size objects.
@@ -716,7 +1074,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 // decryption succeeded.
 //
 // DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
+func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, err error) {
 	// Directories are never encrypted.
 	if info.IsDir {
 		return false, nil
@@ -734,7 +1092,7 @@ func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, e
 			err = errEncryptedObject
 			return
 		}
-		info.Size, err = info.DecryptedSize()
+		_, err = info.DecryptedSize()
 	}
 	return
 }