allow custom speedtest bucket (#15271)

this allows for specifying existing buckets with

- object replication enabled
- object encryption enabled
- object versioning enabled
- object locking enabled
This commit is contained in:
Harshavardhana 2022-07-12 10:12:47 -07:00 committed by GitHub
parent 57d1f31054
commit b4eb74f5ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 130 additions and 98 deletions

View File

@ -1149,17 +1149,17 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
}
}
// SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
a.ObjectSpeedtestHandler(w, r)
// SpeedtestHandler - Deprecated. See ObjectSpeedTestHandler
func (a adminAPIHandlers) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
a.ObjectSpeedTestHandler(w, r)
}
// ObjectSpeedtestHandler - reports maximum speed of a cluster by performing PUT and
// ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
// GET operations on the server, supports auto tuning by default by automatically
// increasing concurrency and stopping when we have reached the limits on the
// system.
func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ObjectSpeedtestHandler")
func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "ObjectSpeedTestHandler")
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@ -1176,8 +1176,9 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
sizeStr := r.Form.Get(peerRESTSize)
durationStr := r.Form.Get(peerRESTDuration)
concurrentStr := r.Form.Get(peerRESTConcurrent)
storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
autotune := r.Form.Get("autotune") == "true"
storageClass := r.Form.Get("storage-class")
size, err := strconv.Atoi(sizeStr)
if err != nil {
@ -1212,15 +1213,23 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
autotune = false
}
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
if customBucket == "" {
customBucket = globalObjectPerfBucket
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, customBucket)
if err != nil {
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if !bucketExists {
defer deleteObjectPerfBucket(objectAPI)
}
}
if !bucketExists {
defer deleteObjectPerfBucket(objectAPI)
}
defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
DeletePrefix: true,
})
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
@ -1232,7 +1241,14 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
defer keepAliveTicker.Stop()
enc := json.NewEncoder(w)
ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
ch := objectSpeedTest(ctx, speedTestOpts{
objectSize: size,
concurrencyStart: concurrent,
duration: duration,
autotune: autotune,
storageClass: storageClass,
bucketName: customBucket,
})
for {
select {
case <-ctx.Done():
@ -1255,9 +1271,8 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
}
}
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) {
err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
if err != nil {
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}); err != nil {
if _, ok := err.(BucketExists); !ok {
// Only BucketExists error can be ignored.
return false, err
@ -2203,9 +2218,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
autotune = false
}
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, globalObjectPerfBucket)
if err != nil {
healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error()
healthInfo.Perf.Error = "Unable to create bucket: " + err.Error()
partialWrite(healthInfo)
return
}
@ -2215,7 +2230,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
}
opts := speedTestOpts{
throughputSize: size,
objectSize: size,
concurrencyStart: concurrent,
duration: 10 * time.Second,
autotune: autotune,

View File

@ -244,8 +244,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
}
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))

View File

@ -27,6 +27,7 @@ import (
"time"
"github.com/minio/console/restapi"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/bucket/bandwidth"
"github.com/minio/minio/internal/config"
@ -376,6 +377,9 @@ var (
// MinIO version unix timestamp
globalVersionUnix uint64
// MinIO client
globalMinioClient *minio.Client
// Add new variable global values here.
)

View File

@ -1512,17 +1512,15 @@ func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration)
return results
}
// Speedtest run GET/PUT tests at input concurrency for requested object size,
// SpeedTest run GET/PUT tests at input concurrency for requested object size,
// optionally you can extend the tests longer with time.Duration.
func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
concurrent int, duration time.Duration, storageClass string,
) []SpeedtestResult {
func (sys *NotificationSys) SpeedTest(ctx context.Context, sopts speedTestOpts) []SpeedTestResult {
length := len(sys.allPeerClients)
if length == 0 {
// For single node erasure setup.
length = 1
}
results := make([]SpeedtestResult, length)
results := make([]SpeedTestResult, length)
scheme := "http"
if globalIsTLS {
@ -1537,8 +1535,7 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
wg.Add(1)
go func(index int) {
defer wg.Done()
r, err := sys.peerClients[index].Speedtest(ctx, size,
concurrent, duration, storageClass)
r, err := sys.peerClients[index].SpeedTest(ctx, sopts)
u := &url.URL{
Scheme: scheme,
Host: sys.peerClients[index].host.String(),
@ -1555,7 +1552,7 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
wg.Add(1)
go func() {
defer wg.Done()
r, err := selfSpeedtest(ctx, size, concurrent, duration, storageClass)
r, err := selfSpeedTest(ctx, sopts)
u := &url.URL{
Scheme: scheme,
Host: globalLocalNodeName,

View File

@ -803,26 +803,25 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
return ch, nil
}
func (client *peerRESTClient) Speedtest(ctx context.Context, size,
concurrent int, duration time.Duration, storageClass string,
) (SpeedtestResult, error) {
func (client *peerRESTClient) SpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) {
values := make(url.Values)
values.Set(peerRESTSize, strconv.Itoa(size))
values.Set(peerRESTConcurrent, strconv.Itoa(concurrent))
values.Set(peerRESTDuration, duration.String())
values.Set(peerRESTStorageClass, storageClass)
values.Set(peerRESTSize, strconv.Itoa(opts.objectSize))
values.Set(peerRESTConcurrent, strconv.Itoa(opts.concurrency))
values.Set(peerRESTDuration, opts.duration.String())
values.Set(peerRESTStorageClass, opts.storageClass)
values.Set(peerRESTBucket, opts.bucketName)
respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedtest, values, nil, -1)
respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedTest, values, nil, -1)
if err != nil {
return SpeedtestResult{}, err
return SpeedTestResult{}, err
}
defer http.DrainBody(respBody)
waitReader, err := waitForHTTPResponse(respBody)
if err != nil {
return SpeedtestResult{}, err
return SpeedTestResult{}, err
}
var result SpeedtestResult
var result SpeedTestResult
err = gob.NewDecoder(waitReader).Decode(&result)
if err != nil {
return result, err

View File

@ -63,7 +63,7 @@ const (
peerRESTMethodUpdateMetacacheListing = "/updatemetacache"
peerRESTMethodGetPeerMetrics = "/peermetrics"
peerRESTMethodLoadTransitionTierConfig = "/loadtransitiontierconfig"
peerRESTMethodSpeedtest = "/speedtest"
peerRESTMethodSpeedTest = "/speedtest"
peerRESTMethodDriveSpeedTest = "/drivespeedtest"
peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
peerRESTMethodReloadPoolMeta = "/reloadpoolmeta"

View File

@ -1125,7 +1125,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
}
}
func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
if !s.IsValid(w, r) {
s.writeErrorResponse(w, errors.New("invalid request"))
return
@ -1141,6 +1141,7 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
durationStr := r.Form.Get(peerRESTDuration)
concurrentStr := r.Form.Get(peerRESTConcurrent)
storageClass := r.Form.Get(peerRESTStorageClass)
bucketName := r.Form.Get(peerRESTBucket)
size, err := strconv.Atoi(sizeStr)
if err != nil {
@ -1159,7 +1160,13 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
done := keepHTTPResponseAlive(w)
result, err := selfSpeedtest(r.Context(), size, concurrent, duration, storageClass)
result, err := selfSpeedTest(r.Context(), speedTestOpts{
objectSize: size,
concurrency: concurrent,
duration: duration,
storageClass: storageClass,
bucketName: bucketName,
})
if err != nil {
result.Error = err.Error()
}
@ -1312,7 +1319,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(httpTraceHdrs(server.UpdateMetacacheListingHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(httpTraceHdrs(server.SpeedTestHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.NetSpeedTestHandler))
subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))

View File

@ -30,15 +30,13 @@ import (
"time"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/minio/madmin-go"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/pkg/randreader"
)
// SpeedtestResult return value of the speedtest function
type SpeedtestResult struct {
// SpeedTestResult return value of the speedtest function
type SpeedTestResult struct {
Endpoint string
Uploads uint64
Downloads uint64
@ -50,10 +48,10 @@ func newRandomReader(size int) io.Reader {
}
// Runs the speedtest on local MinIO process.
func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) {
objAPI := newObjectLayerFn()
if objAPI == nil {
return SpeedtestResult{}, errServerNotInitialized
return SpeedTestResult{}, errServerNotInitialized
}
var errOnce sync.Once
@ -62,48 +60,34 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
var totalBytesWritten uint64
var totalBytesRead uint64
region := globalSite.Region
if region == "" {
region = "us-east-1"
}
client, err := minio.New(globalLocalNodeName, &minio.Options{
Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
Secure: globalIsTLS,
Transport: globalProxyTransport,
Region: region,
})
if err != nil {
return SpeedtestResult{}, err
}
objCountPerThread := make([]uint64, concurrent)
objCountPerThread := make([]uint64, opts.concurrency)
uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
defer uploadsCancel()
go func() {
time.Sleep(duration)
time.Sleep(opts.duration)
uploadsCancel()
}()
objNamePrefix := uuid.New().String() + SlashSeparator
objNamePrefix := pathJoin(speedTest, mustGetUUID())
userMetadata := make(map[string]string)
userMetadata[globalObjectPerfUserMetadata] = "true"
userMetadata[globalObjectPerfUserMetadata] = "true" // Bypass S3 API freeze
popts := minio.PutObjectOptions{
UserMetadata: userMetadata,
DisableContentSha256: true,
DisableMultipart: true,
}
wg.Add(concurrent)
for i := 0; i < concurrent; i++ {
wg.Add(opts.concurrency)
for i := 0; i < opts.concurrency; i++ {
go func(i int) {
defer wg.Done()
for {
reader := newRandomReader(size)
tmpObjName := fmt.Sprintf("%s%d.%d", objNamePrefix, i, objCountPerThread[i])
info, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, tmpObjName, reader, int64(size), minio.PutObjectOptions{
UserMetadata: userMetadata,
DisableContentSha256: true,
DisableMultipart: true,
}) // Bypass S3 API freeze
reader := newRandomReader(opts.objectSize)
tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, objCountPerThread[i]))
info, err := globalMinioClient.PutObject(uploadsCtx, opts.bucketName, tmpObjName, reader, int64(opts.objectSize), popts)
if err != nil {
if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
errOnce.Do(func() {
@ -122,18 +106,21 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
// We already saw write failures, no need to proceed into read's
if retError != "" {
return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
return SpeedTestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
}
downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
defer downloadsCancel()
go func() {
time.Sleep(duration)
time.Sleep(opts.duration)
downloadsCancel()
}()
wg.Add(concurrent)
for i := 0; i < concurrent; i++ {
gopts := minio.GetObjectOptions{}
gopts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze
wg.Add(opts.concurrency)
for i := 0; i < opts.concurrency; i++ {
go func(i int) {
defer wg.Done()
var j uint64
@ -144,9 +131,8 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
if objCountPerThread[i] == j {
j = 0
}
opts := minio.GetObjectOptions{}
opts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze
r, err := client.GetObject(downloadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s%d.%d", objNamePrefix, i, j), opts)
tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, j))
r, err := globalMinioClient.GetObject(downloadsCtx, opts.bucketName, tmpObjName, gopts)
if err != nil {
errResp, ok := err.(minio.ErrorResponse)
if ok && errResp.StatusCode == http.StatusNotFound {
@ -183,7 +169,7 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
}
wg.Wait()
return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
return SpeedTestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
}
// To collect RX stats during "mc support perf net"

View File

@ -35,6 +35,8 @@ import (
"time"
"github.com/minio/cli"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/bandwidth"
"github.com/minio/minio/internal/color"
@ -640,6 +642,18 @@ func serverMain(ctx *cli.Context) {
printStartupMessage(getAPIEndpoints(), err)
}()
region := globalSite.Region
if region == "" {
region = "us-east-1"
}
globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{
Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
Secure: globalIsTLS,
Transport: globalProxyTransport,
Region: region,
})
logger.FatalIf(err, "Unable to initialize MinIO client")
if serverDebugLog {
logger.Info("== DEBUG Mode enabled ==")
logger.Info("Currently set environment settings:")

View File

@ -28,12 +28,16 @@ import (
"github.com/minio/madmin-go"
)
const speedTest = "speedtest"
type speedTestOpts struct {
throughputSize int
objectSize int
concurrencyStart int
concurrency int
duration time.Duration
autotune bool
storageClass string
bucketName string
}
// Get the max throughput and iops numbers.
@ -46,7 +50,7 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
throughputHighestGet := uint64(0)
throughputHighestPut := uint64(0)
var throughputHighestResults []SpeedtestResult
var throughputHighestResults []SpeedTestResult
sendResult := func() {
var result madmin.SpeedTestResult
@ -54,9 +58,9 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
durationSecs := opts.duration.Seconds()
result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(opts.throughputSize) / uint64(durationSecs)
result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(opts.objectSize) / uint64(durationSecs)
result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(opts.throughputSize) / uint64(durationSecs)
result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(opts.objectSize) / uint64(durationSecs)
for i := 0; i < len(throughputHighestResults); i++ {
errStr := ""
if throughputHighestResults[i].Error != "" {
@ -76,18 +80,18 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
Endpoint: throughputHighestResults[i].Endpoint,
ThroughputPerSec: throughputHighestResults[i].Uploads / uint64(durationSecs),
ObjectsPerSec: throughputHighestResults[i].Uploads / uint64(opts.throughputSize) / uint64(durationSecs),
ObjectsPerSec: throughputHighestResults[i].Uploads / uint64(opts.objectSize) / uint64(durationSecs),
Err: errStr,
})
result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
Endpoint: throughputHighestResults[i].Endpoint,
ThroughputPerSec: throughputHighestResults[i].Downloads / uint64(durationSecs),
ObjectsPerSec: throughputHighestResults[i].Downloads / uint64(opts.throughputSize) / uint64(durationSecs),
ObjectsPerSec: throughputHighestResults[i].Downloads / uint64(opts.objectSize) / uint64(durationSecs),
Err: errStr,
})
}
result.Size = opts.throughputSize
result.Size = opts.objectSize
result.Disks = globalEndpoints.NEndpoints()
result.Servers = len(globalNotificationSys.peerClients) + 1
result.Version = Version
@ -104,9 +108,15 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
default:
}
results := globalNotificationSys.Speedtest(ctx,
opts.throughputSize, concurrency,
opts.duration, opts.storageClass)
sopts := speedTestOpts{
objectSize: opts.objectSize,
concurrency: concurrency,
duration: opts.duration,
storageClass: opts.storageClass,
bucketName: opts.bucketName,
}
results := globalNotificationSys.SpeedTest(ctx, sopts)
sort.Slice(results, func(i, j int) bool {
return results[i].Endpoint < results[j].Endpoint
})