Skip to content

Commit

Permalink
pre-parse (log-modules, log-level)
Browse files Browse the repository at this point in the history 
* micro-optimize, reduce code

Signed-off-by: Alex Aizman <[email protected]>
  • Loading branch information
@alex-aizman
alex-aizman committed Jan 9, 2024
1 parent 1c26c94 commit a1a0190
Show file tree
Hide file tree
Showing 71 changed files with 289 additions and 301 deletions.
28 changes: 14 additions & 14 deletions ais/backend/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ func (*awsProvider) HeadBucket(_ ctx, bck *meta.Bck) (bckProps cos.StrKVs, errCo
cloudBck = bck.RemoteBck()
)
svc, region, errC = newClient(sessConf{bck: cloudBck}, "")
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[head_bucket]", cloudBck.Name, errC)
}
if region == "" {
Expand Down Expand Up @@ -146,7 +146,7 @@ func (awsp *awsProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
versioning bool
)
svc, _, err = newClient(sessConf{bck: cloudBck}, "[list_objects]")
if err != nil && verbose {
if err != nil && cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Warningln(err)
}

Expand All @@ -167,7 +167,7 @@ func (awsp *awsProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls

resp, err := svc.ListObjectsV2(params)
if err != nil {
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infoln("list_objects", cloudBck.Name, err)
}
errCode, err = awsErrorToAISError(err, cloudBck, "")
Expand Down Expand Up @@ -205,7 +205,7 @@ func (awsp *awsProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
}

if len(lst.Entries) == 0 || !versioning {
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infoln("[list_objects]", cloudBck.Name, len(lst.Entries))
}
return
Expand Down Expand Up @@ -236,7 +236,7 @@ func (awsp *awsProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
}
}
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infoln("[list_objects]", cloudBck.Name, len(lst.Entries), num)
}
return
Expand All @@ -260,7 +260,7 @@ func (*awsProvider) ListBuckets(cmn.QueryBcks) (bcks cmn.Bcks, errCode int, err

bcks = make(cmn.Bcks, len(result.Buckets))
for idx, bck := range result.Buckets {
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infoln("[bucket_names]", aws.StringValue(bck.Name), "created", *bck.CreationDate)
}
bcks[idx] = cmn.Bck{
Expand All @@ -283,7 +283,7 @@ func (*awsProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int
cloudBck = lom.Bck().RemoteBck()
)
svc, _, err = newClient(sessConf{bck: cloudBck}, "[head_object]")
if err != nil && verbose {
if err != nil && cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Warningln(err)
}
headOutput, err = svc.HeadObject(&s3.HeadObjectInput{
Expand Down Expand Up @@ -334,7 +334,7 @@ func (*awsProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int
}
oa.SetCustomKey(cmn.LastModified, fmtTime(mtime))
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[head_object]", cloudBck.Cname(lom.ObjName))
}
return
Expand All @@ -352,7 +352,7 @@ func (awsp *awsProvider) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT)
params := allocPutParams(res, owt)
err := awsp.t.PutObject(lom, params)
core.FreePutParams(params)
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[get_object]", lom.String(), err)
}
return 0, err
Expand All @@ -364,7 +364,7 @@ func (*awsProvider) GetObjReader(ctx context.Context, lom *core.LOM) (res core.G
cloudBck = lom.Bck().RemoteBck()
)
svc, _, err := newClient(sessConf{bck: cloudBck}, "[get_object]")
if err != nil && superVerbose {
if err != nil && cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Warningln(err)
}
obj, err = svc.GetObjectWithContext(ctx, &s3.GetObjectInput{
Expand Down Expand Up @@ -429,7 +429,7 @@ func (*awsProvider) PutObj(r io.ReadCloser, lom *core.LOM) (errCode int, err err
)

svc, _, err = newClient(sessConf{bck: cloudBck}, "[put_object]")
if err != nil && superVerbose {
if err != nil && cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Warningln(err)
}

Expand Down Expand Up @@ -460,7 +460,7 @@ func (*awsProvider) PutObj(r io.ReadCloser, lom *core.LOM) (errCode int, err err
lom.SetCustomKey(cmn.MD5ObjMD, v)
}
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[put_object]", lom.String())
}
cos.Close(r)
Expand All @@ -477,7 +477,7 @@ func (*awsProvider) DeleteObj(lom *core.LOM) (errCode int, err error) {
cloudBck = lom.Bck().RemoteBck()
)
svc, _, err = newClient(sessConf{bck: cloudBck}, "[delete_object]")
if err != nil && verbose {
if err != nil && cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Warningln(err)
}
_, err = svc.DeleteObject(&s3.DeleteObjectInput{
Expand All @@ -488,7 +488,7 @@ func (*awsProvider) DeleteObj(lom *core.LOM) (errCode int, err error) {
errCode, err = awsErrorToAISError(err, cloudBck, lom.ObjName)
return
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[delete_object]", lom.String())
}
return
Expand Down
10 changes: 5 additions & 5 deletions ais/backend/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func (ap *azureProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
marker = azblob.Marker{}
opts = azblob.ListBlobsSegmentOptions{Prefix: msg.Prefix, MaxResults: int32(msg.PageSize)}
)
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("list_objects %s", cloudBck.Name)
}
if msg.ContinuationToken != "" {
Expand Down Expand Up @@ -276,7 +276,7 @@ func (ap *azureProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
if resp.NextMarker.Val != nil {
lst.ContinuationToken = *resp.NextMarker.Val
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[list_objects] count %d(marker: %s)", len(lst.Entries), lst.ContinuationToken)
}
return
Expand Down Expand Up @@ -342,7 +342,7 @@ func (ap *azureProvider) HeadObj(ctx context.Context, lom *core.LOM) (oa *cmn.Ob
if v, ok := h.EncodeCksum(resp.ContentMD5()); ok {
oa.SetCustomKey(cmn.MD5ObjMD, v)
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("[head_object] %s", lom)
}
return
Expand All @@ -360,7 +360,7 @@ func (ap *azureProvider) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT)
params := allocPutParams(res, owt)
err := ap.t.PutObject(lom, params)
core.FreePutParams(params)
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[get_object]", lom.String(), err)
}
return 0, err
Expand Down Expand Up @@ -475,7 +475,7 @@ func (ap *azureProvider) PutObj(r io.ReadCloser, lom *core.LOM) (int, error) {
lom.SetCustomKey(cmn.ETag, v) // NOTE: using ETag as version
lom.SetVersion(v)
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("[put_object] %s", lom)
}
return http.StatusOK, nil
Expand Down
8 changes: 0 additions & 8 deletions ais/backend/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,12 @@ import (
"time"

"github.com/NVIDIA/aistore/cmn"
"github.com/NVIDIA/aistore/cmn/cos"
"github.com/NVIDIA/aistore/core"
"github.com/NVIDIA/aistore/fs"
)

type ctx = context.Context // used when omitted for shortness sake

var verbose, superVerbose bool

func Init(config *cmn.Config) {
verbose = config.FastV(4, cos.SmoduleBackend)
superVerbose = config.FastV(5, cos.SmoduleBackend)
}

func fmtTime(t time.Time) string { return t.Format(time.RFC3339) }

func calcPageSize(pageSize, maxPageSize uint) uint {
Expand Down
16 changes: 8 additions & 8 deletions ais/backend/gcp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (*gcpProvider) CreateBucket(_ *meta.Bck) (int, error) {
//

func (*gcpProvider) HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps cos.StrKVs, errCode int, err error) {
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("head_bucket %s", bck.Name)
}
cloudBck := bck.RemoteBck()
Expand Down Expand Up @@ -171,7 +171,7 @@ func (gcpp *gcpProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
)
nextPageToken, errPage := pager.NextPage(&objs)
if errPage != nil {
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("list_objects %s: %v", cloudBck.Name, errPage)
}
errCode, err = gcpErrorToAISError(errPage, cloudBck)
Expand Down Expand Up @@ -208,7 +208,7 @@ func (gcpp *gcpProvider) ListObjects(bck *meta.Bck, msg *apc.LsoMsg, lst *cmn.Ls
}
}
lst.Entries = lst.Entries[:l]
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[list_objects] count %d", len(lst.Entries))
}
return
Expand Down Expand Up @@ -242,7 +242,7 @@ func (gcpp *gcpProvider) ListBuckets(_ cmn.QueryBcks) (bcks cmn.Bcks, errCode in
Name: battrs.Name,
Provider: apc.GCP,
})
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[bucket_names] %s: created %v, versioning %t",
battrs.Name, battrs.Created, battrs.VersioningEnabled)
}
Expand Down Expand Up @@ -292,7 +292,7 @@ func (*gcpProvider) HeadObj(ctx context.Context, lom *core.LOM) (oa *cmn.ObjAttr
// unlike other custom attrs, "Content-Type" is not getting stored w/ LOM
// - only shown via list-objects and HEAD when not present
oa.SetCustomKey(cos.HdrContentType, attrs.ContentType)
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("[head_object] %s", cloudBck.Cname(lom.ObjName))
}
return
Expand All @@ -310,7 +310,7 @@ func (gcpp *gcpProvider) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT)
params := allocPutParams(res, owt)
err := gcpp.t.PutObject(lom, params)
core.FreePutParams(params)
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infoln("[get_object]", lom.String(), err)
}
return 0, err
Expand Down Expand Up @@ -402,7 +402,7 @@ func (gcpp *gcpProvider) PutObj(r io.ReadCloser, lom *core.LOM) (errCode int, er
return
}
_ = setCustomGs(lom, attrs)
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("[put_object] %s, size %d", lom, written)
}
return
Expand All @@ -421,7 +421,7 @@ func (*gcpProvider) DeleteObj(lom *core.LOM) (errCode int, err error) {
errCode, err = handleObjectError(gctx, gcpClient, err, cloudBck)
return
}
if superVerbose {
if cmn.Rom.FastV(5, cos.SmoduleBackend) {
nlog.Infof("[delete_object] %s", lom)
}
return
Expand Down
8 changes: 4 additions & 4 deletions ais/backend/hdfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func (hp *hdfsProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode
oa = &cmn.ObjAttrs{}
oa.SetCustomKey(cmn.SourceObjMD, apc.HDFS)
oa.Size = fr.Stat().Size()
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[head_object] %s", lom)
}
return
Expand All @@ -237,7 +237,7 @@ func (hp *hdfsProvider) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT)
}
params := allocPutParams(res, owt)
err := hp.t.PutObject(lom, params)
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infoln("[get_object]", lom.String(), err)
}
return 0, err
Expand Down Expand Up @@ -295,7 +295,7 @@ finish:
errCode, err = hdfsErrorToAISError(err)
return errCode, err
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[put_object] %s", lom)
}

Expand All @@ -312,7 +312,7 @@ func (hp *hdfsProvider) DeleteObj(lom *core.LOM) (errCode int, err error) {
errCode, err = hdfsErrorToAISError(err)
return errCode, err
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[delete_object] %s", lom)
}
return 0, nil
Expand Down
12 changes: 6 additions & 6 deletions ais/backend/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (hp *httpProvider) HeadBucket(ctx context.Context, bck *meta.Bck) (bckProps
return nil, http.StatusBadRequest, err
}

if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[head_bucket] original_url: %q", origURL)
}

Expand Down Expand Up @@ -118,7 +118,7 @@ func (hp *httpProvider) HeadObj(ctx context.Context, lom *core.LOM) (oa *cmn.Obj
origURL, err := getOriginalURL(ctx, bck, lom.ObjName)
debug.AssertNoErr(err)

if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[head_object] original_url: %q", origURL)
}
resp, err := hp.client(origURL).Head(origURL)
Expand All @@ -137,7 +137,7 @@ func (hp *httpProvider) HeadObj(ctx context.Context, lom *core.LOM) (oa *cmn.Obj
if v, ok := h.EncodeVersion(resp.Header.Get(cos.HdrETag)); ok {
oa.SetCustomKey(cmn.ETag, v)
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[head_object] %s", lom)
}
return
Expand All @@ -154,7 +154,7 @@ func (hp *httpProvider) GetObj(ctx context.Context, lom *core.LOM, owt cmn.OWT)
if res.Err != nil {
return 0, res.Err
}
if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[get_object] %s", lom)
}
return 0, nil
Expand All @@ -170,7 +170,7 @@ func (hp *httpProvider) GetObjReader(ctx context.Context, lom *core.LOM) (res co
origURL, err := getOriginalURL(ctx, bck, lom.ObjName)
debug.AssertNoErr(err)

if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[HTTP CLOUD][GET] original_url: %q", origURL)
}

Expand All @@ -185,7 +185,7 @@ func (hp *httpProvider) GetObjReader(ctx context.Context, lom *core.LOM) (res co
return
}

if verbose {
if cmn.Rom.FastV(4, cos.SmoduleBackend) {
nlog.Infof("[HTTP CLOUD][GET] success, size: %d", resp.ContentLength)
}

Expand Down
2 changes: 1 addition & 1 deletion ais/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ func initDaemon(version, buildTime string) cos.Runner {

// reg xaction factories
xs.Xreg(false /* x-ele only */)
space.Xreg(config)
space.Xreg()

t := newTarget(co)
t.init(config)
Expand Down
Loading

0 comments on commit a1a0190

Please sign in to comment.