text:
// Package ais provides core functionality for the AIStore object storage.
/*
 * Copyright (c) 2018-2020, NVIDIA CORPORATION. All rights reserved.
 */
package ais

import (
	"context"
	"encoding"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
	"unsafe"

	"github.com/NVIDIA/aistore/3rdparty/glog"
	"github.com/NVIDIA/aistore/cluster"
	"github.com/NVIDIA/aistore/cmn"
	"github.com/NVIDIA/aistore/cmn/debug"
	"github.com/NVIDIA/aistore/ec"
	"github.com/NVIDIA/aistore/fs"
	"github.com/NVIDIA/aistore/memsys"
	"github.com/NVIDIA/aistore/stats"
	"github.com/NVIDIA/aistore/transport"
	"github.com/NVIDIA/aistore/xaction/xreg"
)

//
// PUT, GET, APPEND, and COPY object
//

type (
	putObjInfo struct {
		started time.Time // started time of receiving - used to calculate the recv duration
		t       *targetrunner
		lom     *cluster.LOM
		// Reader with the content of the object.
		r io.ReadCloser
		// If available (not `none`), can be validated and will be stored with the object
		// see `writeToFile` method
		cksumToUse *cmn.Cksum
		// object size aka Content-Length
		size int64
		// Context used when putting the object which should be contained in
		// cloud bucket. It usually contains credentials to access the cloud.
		ctx context.Context
		// FQN which is used only temporarily for receiving file. After
		// successful receive is renamed to actual FQN.
		workFQN string
		// Determines the receive type of the request.
		recvType cluster.RecvType
		// if true, poi won't erasure-encode an object when finalizing
		skipEC bool
	}

	getObjInfo struct {
		started time.Time // started time of receiving - used to calculate the recv duration
		t       *targetrunner
		lom     *cluster.LOM
		// Writer where the object will be written.
		w io.Writer
		// Context used when receiving the object which is contained in cloud
		// bucket. It usually contains credentials to access the cloud.
		ctx context.Context
		// Contains object range query
		ranges cmn.RangesQuery
		// Determines if it is GFN request
		isGFN bool
		// true: chunked transfer (en)coding as per https://tools.ietf.org/html/rfc7230#page-36
		chunked bool
	}

	// Contains information packed in append handle.
	handleInfo struct {
		nodeID       string
		filePath     string
		partialCksum *cmn.CksumHash
	}

	appendObjInfo struct {
		started time.Time // started time of receiving - used to calculate the recv duration
		t       *targetrunner
		lom     *cluster.LOM

		// Reader with the content of the object.
		r io.ReadCloser
		// Object size aka Content-Length.
		size int64
		// Append/Flush operation.
		op string
		hi handleInfo // Information contained in handle.

		cksum *cmn.Cksum // Expected checksum of the final object.
	}

	copyObjInfo struct {
		cluster.CopyObjectParams
		t         *targetrunner
		localOnly bool // copy locally with no HRW=>target
		finalize  bool // copies and EC (as in poi.finalize())
	}
)

////////////////
// PUT OBJECT //
////////////////

func (poi *putObjInfo) putObject() (errCode int, err error) {
	debug.Assert(cluster.RegularPut <= poi.recvType && poi.recvType <= cluster.Migrated)

	lom := poi.lom
	// optimize out if the checksums do match
	if !poi.cksumToUse.IsEmpty() {
		if lom.Cksum().Equal(poi.cksumToUse) {
			if glog.FastV(4, glog.SmoduleAIS) {
				glog.Infof("%s is valid %s: PUT is a no-op", lom, poi.cksumToUse)
			}
			cmn.DrainReader(poi.r)
			return 0, nil
		}
	}

	if !daemon.dryRun.disk {
		if err := poi.writeToFile(); err != nil {
			return http.StatusInternalServerError, err
		}
		if errCode, err := poi.finalize(); err != nil {
			return errCode, err
		}
	}
	if poi.recvType == cluster.RegularPut {
		delta := time.Since(poi.started)
		poi.t.statsT.AddMany(
			stats.NamedVal64{Name: stats.PutCount, Value: 1},
			stats.NamedVal64{Name: stats.PutLatency, Value: int64(delta)},
		)
		if glog.FastV(4, glog.SmoduleAIS) {
			glog.Infof("PUT %s: %s", lom, delta)
		}
	}
	return 0, nil
}

func (poi *putObjInfo) finalize() (errCode int, err error) {
	if errCode, err = poi.tryFinalize(); err != nil {
		if err1 := fs.Access(poi.workFQN); err1 == nil || !os.IsNotExist(err1) {
			if err1 == nil {
				err1 = err
			}
			poi.t.fsErr(err1, poi.workFQN)
			if err2 := cmn.RemoveFile(poi.workFQN); err2 != nil {
				glog.Errorf("Nested error: %s => (remove %s => err: %v)", err1, poi.workFQN, err2)
			}
		}
		poi.lom.Uncache(true /*delDirty*/)
		return
	}
	if !poi.skipEC {
		if ecErr := ec.ECM.EncodeObject(poi.lom); ecErr != nil && ecErr != ec.ErrorECDisabled {
			err = ecErr
			return
		}
	}

	poi.t.putMirror(poi.lom)
	return
}

// poi.workFQN => LOM
func (poi *putObjInfo) tryFinalize() (errCode int, err error) {
	var (
		lom = poi.lom
		bck = lom.Bck()
		bmd = poi.t.owner.bmd.Get()
	)
	// remote versioning
	if bck.IsRemote() && poi.recvType == cluster.RegularPut {
		var version string
		if bck.IsRemoteAIS() {
			version, errCode, err = poi.putRemoteAIS()
		} else {
			version, errCode, err = poi.putCloud()
		}
		if err != nil {
			glog.Errorf("PUT %s: %v", lom, err)
			return
		}
		if lom.VersionConf().Enabled {
			lom.SetVersion(version)
		}
	}
	if _, present := bmd.Get(bck); !present {
		err = fmt.Errorf("PUT %s: bucket %s does not exist", lom, bck)
		errCode = http.StatusBadRequest
		return
	}
	if poi.recvType == cluster.ColdGet {
		debug.Assert(!lom.TryLock(true)) // cold GET: caller must take a lock
	} else {
		lom.Lock(true)
		defer lom.Unlock(true)
	}
	// ais versioning
	if bck.IsAIS() && lom.VersionConf().Enabled {
		// NOTE: the caller is expected to load it and get the current version, if exists
		if poi.recvType == cluster.RegularPut || lom.Version(true) == "" {
			if err = lom.IncVersion(); err != nil {
				return
			}
		}
	}
	if err = cmn.Rename(poi.workFQN, lom.FQN); err != nil {
		err = fmt.Errorf("PUT %s: failed to rename: %w", lom, err)
		return
	}
	if lom.HasCopies() {
		// TODO: recover
		if errdc := lom.DelAllCopies(); errdc != nil {
			glog.Errorf("PUT %s: failed to delete old copies [%v], proceeding to PUT anyway...", lom, errdc)
		}
	}
	err = lom.Persist(true)
	if err != nil {
		lom.Uncache(true /*delDirty*/)
	}
	return
}

func (poi *putObjInfo) putCloud() (version string, errCode int, err error) {
	var (
		lom = poi.lom
		bck = lom.Bck()
	)
	file, err := os.Open(poi.workFQN)
	if err != nil {
		err = fmt.Errorf("failed to open %s err: %w", poi.workFQN, err)
		return
	}

	cloud := poi.t.Backend(bck)
	customMD := cmn.SimpleKVs{
		cluster.SourceObjMD: cloud.Provider(),
	}

	version, errCode, err = cloud.PutObj(poi.ctx, file, lom)
	if version != "" {
		customMD[cluster.VersionObjMD] = version
	}
	lom.SetCustomMD(customMD)
	cmn.Close(file)
	return
}

func (poi *putObjInfo) putRemoteAIS() (version string, errCode int, err error) {
	var (
		lom = poi.lom
		bck = lom.Bck()
	)
	cmn.Assert(bck.IsRemoteAIS())
	fh, errOpen := cmn.NewFileHandle(poi.workFQN) // Closed by `PutObj`.
	if errOpen != nil {
		err = fmt.Errorf("failed to open %s err: %w", poi.workFQN, errOpen)
		return
	}
	version, errCode, err = poi.t.Backend(bck).PutObj(poi.ctx, fh, lom)
	return
}

// NOTE: LOM is updated on the end of the call with proper size and checksum.
// NOTE: `roi.r` is closed on the end of the call.
func (poi *putObjInfo) writeToFile() (err error) {
	var (
		written int64
		file    *os.File
		buf     []byte
		slab    *memsys.Slab
		reader  = poi.r
		writer  io.Writer
		writers = make([]io.Writer, 0, 4)
		cksums  = struct {
			store *cmn.CksumHash // store with LOM
			given *cmn.CksumHash // compute additionally
			expct *cmn.Cksum     // and validate against `expct` if required/available
		}{}
		conf = poi.lom.CksumConf()
	)
	if daemon.dryRun.disk {
		return
	}
	if file, err = poi.lom.CreateFile(poi.workFQN); err != nil {
		return
	}
	writer = cmn.WriterOnly{Writer: file} // Hiding `ReadFrom` for `*os.File` introduced in Go1.15.
	if poi.size == 0 {
		buf, slab = poi.t.gmm.Alloc()
	} else {
		buf, slab = poi.t.gmm.Alloc(poi.size)
	}
	// cleanup
	defer func() { // free & cleanup on err
		slab.Free(buf)
		cmn.Close(reader)

		if err != nil {
			if nestedErr := file.Close(); nestedErr != nil {
				glog.Errorf("Nested (%v): failed to close received object %s, err: %v",
					err, poi.workFQN, nestedErr)
			}
			if nestedErr := cmn.RemoveFile(poi.workFQN); nestedErr != nil {
				glog.Errorf("Nested (%v): failed to remove %s, err: %v", err, poi.workFQN, nestedErr)
			}
		}
	}()
	// checksums
	if conf.Type == cmn.ChecksumNone {
		poi.lom.SetCksum(cmn.NoneCksum)
		goto write
	}
	if poi.recvType == cluster.Migrated && !conf.ShouldValidate() && !poi.cksumToUse.IsEmpty() {
		// if migration validation is not configured we can just take
		// the checksum that has arrived with the object (and compute it if not present)
		poi.lom.SetCksum(poi.cksumToUse)
		goto write
	}

	// compute checksum and save it as part of the object metadata
	cksums.store = cmn.NewCksumHash(conf.Type)
	writers = append(writers, cksums.store.H)
	if conf.ShouldValidate() && !poi.cksumToUse.IsEmpty() {
		// if validate-cold-get and the cksum is provided we should also check md5 hash (aws, gcp)
		// or if the object is migrated, and `conf.ValidateObjMove` we should check with existing checksum
		cksums.expct = poi.cksumToUse
		cksums.given = cmn.NewCksumHash(poi.cksumToUse.Type())
		writers = append(writers, cksums.given.H)
	}

write:
	if len(writers) == 0 {
		written, err = io.CopyBuffer(writer, reader, buf)
	} else {
		writers = append(writers, writer)
		written, err = io.CopyBuffer(cmn.NewWriterMulti(writers...), reader, buf)
	}
	if err != nil {
		return
	}
	// validate
	if cksums.given != nil {
		cksums.given.Finalize()
		if !cksums.given.Equal(cksums.expct) {
			err = cmn.NewBadDataCksumError(cksums.expct, &cksums.given.Cksum, poi.lom.String())
			poi.t.statsT.AddMany(
				stats.NamedVal64{Name: stats.ErrCksumCount, Value: 1},
				stats.NamedVal64{Name: stats.ErrCksumSize, Value: written},
			)
			return
		}
	}
	// ok
	poi.lom.SetSize(written)
	if cksums.store != nil {
		cksums.store.Finalize()
		poi.lom.SetCksum(&cksums.store.Cksum)
	}
	if err = file.Close(); err != nil {
		return fmt.Errorf("failed to close received %s: %w", poi.workFQN, err)
	}
	return nil
}

////////////////
// GET OBJECT //
////////////////

func (goi *getObjInfo) getObject() (sent bool, errCode int, err error) {
	var (
		cs                                            fs.CapStatus
		doubleCheck, retry, retried, coldGet, capRead bool
	)
	// under lock: lom init, restore from cluster
	goi.lom.Lock(false)
do:
	// all subsequent checks work with disks - skip all if dryRun.disk=true
	if daemon.dryRun.disk {
		goto get
	}
	err = goi.lom.Load()
	if err != nil {
		coldGet = cmn.IsObjNotExist(err)
		if !coldGet {
			goi.lom.Unlock(false)
			return sent, http.StatusInternalServerError, err
		}
		capRead = true // set flag to avoid calling later
		cs = fs.GetCapStatus()
		if cs.OOS {
			// Immediate return for no space left to restore object.
			goi.lom.Unlock(false)
			return sent, http.StatusInsufficientStorage, cs.Err
		}
	}

	if coldGet && goi.lom.Bck().IsAIS() {
		// try lookup and restore
		goi.lom.Unlock(false)
		doubleCheck, errCode, err = goi.tryRestoreObject()
		if doubleCheck && err != nil {
			lom2 := &cluster.LOM{ObjName: goi.lom.ObjName}
			er2 := lom2.Init(goi.lom.Bucket())
			if er2 == nil {
				er2 = lom2.Load()
				if er2 == nil {
					goi.lom = lom2
					err = nil
				}
			}
		}
		if err != nil {
			return
		}
		goi.lom.Lock(false)
		goto get
	}
	// exists && remote|cloud: check ver if requested
	if !coldGet && goi.lom.Bck().IsRemote() {
		if goi.lom.Version() != "" && goi.lom.VersionConf().ValidateWarmGet {
			goi.lom.Unlock(false)
			if coldGet, errCode, err = goi.t.CheckRemoteVersion(goi.ctx, goi.lom); err != nil {
				goi.lom.Uncache(true /*delDirty*/)
				return
			}
			goi.lom.Lock(false)
		}
	}

	// checksum validation, if requested
	if !coldGet && goi.lom.CksumConf().ValidateWarmGet {
		coldGet, errCode, err = goi.tryRecoverObject()
		if err != nil {
			if !coldGet {
				goi.lom.Unlock(false)
				glog.Error(err)
				return
			}
			glog.Errorf("%v - proceeding to execute cold GET from %s", err, goi.lom.Bck())
		}
	}

	// 3. coldget
	if coldGet {
		if !capRead {
			capRead = true
			cs = fs.GetCapStatus()
		}
		if cs.OOS {
			// No space left to prefetch object.
			goi.lom.Unlock(false)
			return sent, http.StatusInsufficientStorage, cs.Err
		}
		goi.lom.SetAtimeUnix(goi.started.UnixNano())
		if errCode, err := goi.t.GetCold(goi.ctx, goi.lom, cluster.GetCold); err != nil {
			return sent, errCode, err
		}
		goi.t.putMirror(goi.lom)
	}

	// 4. get locally and stream back
get:
	retry, sent, errCode, err = goi.finalize(coldGet)
	if retry && !retried {
		glog.Warningf("GET %s: uncaching and retrying...", goi.lom)
		retried = true
		goi.lom.Uncache(true /*delDirty*/)
		goto do
	}

	goi.lom.Unlock(false)
	return
}

// validate checksum; if corrupted try to recover from other replicas or EC slices
func (goi *getObjInfo) tryRecoverObject() (coldGet bool, code int, err error) {
	var (
		lom     = goi.lom
		retried bool
	)
retry:
	err = lom.ValidateMetaChecksum()
	if err == nil {
		err = lom.ValidateContentChecksum()
	}
	if err == nil {
		return
	}
	code = http.StatusInternalServerError
	if _, ok := err.(*cmn.BadCksumError); !ok {
		return
	}
	if !lom.Bck().IsAIS() {
		coldGet = true
		return
	}

	glog.Warning(err)
	redundant := lom.HasCopies() || lom.Bprops().EC.Enabled
	//
	// return err if there's no redundancy OR already recovered once (and failed)
	//
	if retried || !redundant {
		//
		// TODO: mark `deleted` and postpone actual deletion
		//
		if erl := lom.Remove(); erl != nil {
			glog.Warningf("%s: failed to remove corrupted %s, err: %v", goi.t.si, lom, erl)
		}
		return
	}
	//
	// try to recover from BAD CHECKSUM
	//
	cmn.RemoveFile(lom.FQN) // TODO: ditto

	if lom.HasCopies() {
		retried = true
		goi.lom.Unlock(false)
		// lookup and restore the object from local replicas
		restored := lom.RestoreObjectFromAny()
		goi.lom.Lock(false)
		if restored {
			glog.Warningf("%s: recovered corrupted %s from local replica", goi.t.si, lom)
			code = 0
			goto retry
		}
	}
	if lom.Bprops().EC.Enabled {
		retried = true
		goi.lom.Unlock(false)
		cmn.RemoveFile(lom.FQN)
		_, code, err = goi.tryRestoreObject()
		goi.lom.Lock(false)
		if err == nil {
			glog.Warningf("%s: recovered corrupted %s from EC slices", goi.t.si, lom)
			code = 0
			goto retry
		}
	}

	// TODO: ditto
	if erl := lom.Remove(); erl != nil {
		glog.Warningf("%s: failed to remove corrupted %s, err: %v", goi.t.si, lom, erl)
	}
	return
}

// an attempt to restore an object that is missing in the ais bucket - from:
// 1) local FS
// 2) other FSes or targets when resilvering (rebalancing) is running (aka GFN)
// 3) other targets if the bucket erasure coded
// 4) Cloud
func (goi *getObjInfo) tryRestoreObject() (doubleCheck bool, errCode int, err error) {
	var (
		tsi, gfnNode         *cluster.Snode
		smap                 = goi.t.owner.smap.get()
		tname                = goi.t.si.String()
		marked               = xreg.GetResilverMarked()
		interrupted, running = marked.Interrupted, marked.Xact != nil
		gfnActive            = goi.t.gfn.local.active()
		ecEnabled            = goi.lom.Bprops().EC.Enabled
	)
	tsi, err = cluster.HrwTarget(goi.lom.Uname(), &smap.Smap, true /*include maintenance*/)
	if err != nil {
		return
	}
	if interrupted || running || gfnActive {
		if goi.lom.RestoreObjectFromAny() { // get-from-neighbor local (mountpaths) variety
			if glog.FastV(4, glog.SmoduleAIS) {
				glog.Infof("%s restored", goi.lom)
			}
			return
		}
		doubleCheck = running
	}

	// FIXME: if there're not enough EC targets to restore a "sliced" object,
	// we might be able to restore it if it was replicated. In this case even
	// just one additional target might be sufficient. This won't succeed if
	// an object was sliced, neither will ecmanager.RestoreObject(lom)
	enoughECRestoreTargets := goi.lom.Bprops().EC.RequiredRestoreTargets() <= smap.CountActiveTargets()

	// cluster-wide lookup ("get from neighbor")
	marked = xreg.GetRebMarked()
	interrupted, running = marked.Interrupted, marked.Xact != nil
	if running {
		doubleCheck = true
	}
	gfnActive = goi.t.gfn.global.active()
	if running && tsi.ID() != goi.t.si.ID() {
		if goi.t.LookupRemoteSingle(goi.lom, tsi) {
			gfnNode = tsi
			goto gfn
		}
	}
	if running || !enoughECRestoreTargets || ((interrupted || gfnActive) && !ecEnabled) {
		gfnNode = goi.t.lookupRemoteAll(goi.lom, smap)
	}

gfn:
	if gfnNode != nil {
		if goi.getFromNeighbor(goi.lom, gfnNode) {
			if glog.FastV(4, glog.SmoduleAIS) {
				glog.Infof("%s: GFN %s <= %s", tname, goi.lom, gfnNode)
			}
			return
		}
	}

	// restore from existing EC slices if possible
	if ecErr := ec.ECM.RestoreObject(goi.lom); ecErr == nil {
		ecErr = goi.lom.Load(true)
		debug.AssertNoErr(ecErr)
		if ecErr == nil {
			if glog.FastV(4, glog.SmoduleAIS) {
				glog.Infof("%s: EC-recovered %s", tname, goi.lom)
			}
			return
		}
		err = fmt.Errorf("%s: failed to load EC-recovered %s: %v", tname, goi.lom, ecErr)
	} else if ecErr != ec.ErrorECDisabled {
		err = fmt.Errorf("%s: failed to EC-recover %s: %v", tname, goi.lom, ecErr)
	}

	s := fmt.Sprintf("GET local: %s(%s) %s", goi.lom, goi.lom.FQN, cmn.DoesNotExist)
	if err != nil {
		err = fmt.Errorf("%s => [%v]", s, err)
	} else {
		err = errors.New(s)
	}
	errCode = http.StatusNotFound
	return
}

func (goi *getObjInfo) getFromNeighbor(lom *cluster.LOM, tsi *cluster.Snode) (ok bool) {
	header := make(http.Header)
	header.Add(cmn.HeaderCallerID, goi.t.SID())
	header.Add(cmn.HeaderCallerName, goi.t.Sname())
	query := url.Values{}
	query.Set(cmn.URLParamIsGFNRequest, "true")
	query = cmn.AddBckToQuery(query, lom.Bucket())
	reqArgs := cmn.ReqArgs{
		Method: http.MethodGet,
		Base:   tsi.URL(cmn.NetworkIntraData),
		Header: header,
		Path:   cmn.URLPathObjects.Join(lom.BckName(), lom.ObjName),
		Query:  query,
	}
	req, _, cancel, err := reqArgs.ReqWithTimeout(cmn.GCO.Get().Timeout.SendFile)
	if err != nil {
		glog.Errorf("failed to create request, err: %v", err)
		return
	}
	defer cancel()

	resp, err := goi.t.client.data.Do(req) // nolint:bodyclose // closed by `poi.putObject`
	if err != nil {
		glog.Errorf("GFN failure, URL %q, err: %v", reqArgs.URL(), err)
		return
	}
	lom.FromHTTPHdr(resp.Header)
	workFQN := fs.CSM.GenContentFQN(lom, fs.WorkfileType, fs.WorkfileRemote)
	poi := &putObjInfo{
		t:        goi.t,
		lom:      lom,
		r:        resp.Body,
		recvType: cluster.Migrated,
		workFQN:  workFQN,
	}
	if _, err := poi.putObject(); err != nil {
		glog.Error(err)
		return
	}
	ok = true
	return
}

func (goi *getObjInfo) finalize(coldGet bool) (retry, sent bool, errCode int, err error) {
	var (
		file    *os.File
		sgl     *memsys.SGL
		slab    *memsys.Slab
		buf     []byte
		reader  io.Reader
		hdr     http.Header // if it is http request we will write also header
		written int64
	)
	defer func() {
		if file != nil {
			cmn.Close(file)
		}
		if buf != nil {
			slab.Free(buf)
		}
		if sgl != nil {
			sgl.Free()
		}
	}()

	// loopback if disk IO is disabled
	if daemon.dryRun.disk {
		if err = cmn.FloodWriter(goi.w, daemon.dryRun.size); err != nil {
			err = fmt.Errorf("dry-run: failed to send random response, err: %v", err)
			errCode = http.StatusInternalServerError
			goi.t.statsT.Add(stats.ErrGetCount, 1)
			return
		}
		delta := time.Since(goi.started)
		goi.t.statsT.AddMany(
			stats.NamedVal64{Name: stats.GetCount, Value: 1},
			stats.NamedVal64{Name: stats.GetLatency, Value: int64(delta)},
		)
		return
	}

	if rw, ok := goi.w.(http.ResponseWriter); ok {
		hdr = rw.Header()
	}

	fqn := goi.lom.FQN
	if !coldGet && !goi.isGFN {
		// best-effort GET load balancing (see also mirror.findLeastUtilized())
		fqn = goi.lom.LoadBalanceGET()
	}
	file, err = os.Open(fqn)
	if err != nil {
		if os.IsNotExist(err) {
			errCode = http.StatusNotFound
			retry = true // (!lom.IsAIS() || lom.ECEnabled() || GFN...)
		} else {
			goi.t.fsErr(err, fqn)
			err = fmt.Errorf("%s: %w", goi.lom, err)
			errCode = http.StatusInternalServerError
		}
		return
	}

	var (
		r    *cmn.HTTPRange
		size = goi.lom.Size()
	)
	if goi.ranges.Size > 0 {
		size = goi.ranges.Size
	}

	if hdr != nil {
		ranges, err := cmn.ParseMultiRange(goi.ranges.Range, size)
		if err != nil {
			if err == cmn.ErrNoOverlap {
				hdr.Set(cmn.HeaderContentRange, fmt.Sprintf("%s*/%d", cmn.HeaderContentRangeValPrefix, size))
			}
			return false, sent, http.StatusRequestedRangeNotSatisfiable, err
		}

		if len(ranges) > 0 {
			if len(ranges) > 1 {
				err = fmt.Errorf("multi-range is not supported")
				errCode = http.StatusRequestedRangeNotSatisfiable
				return false, sent, errCode, err
			}
			r = &ranges[0]

			hdr.Set(cmn.HeaderAcceptRanges, "bytes")
			hdr.Set(cmn.HeaderContentRange, r.ContentRange(size))
		}
	}

	cksumConf := goi.lom.CksumConf()
	cksumRange := cksumConf.Type != cmn.ChecksumNone && r != nil && cksumConf.EnableReadRange

	if hdr != nil {
		if !goi.lom.Cksum().IsEmpty() && !cksumRange {
			cksumType, cksumValue := goi.lom.Cksum().Get()
			hdr.Set(cmn.HeaderObjCksumType, cksumType)
			hdr.Set(cmn.HeaderObjCksumVal, cksumValue)
		}
		if goi.lom.Version() != "" {
			hdr.Set(cmn.HeaderObjVersion, goi.lom.Version())
		}
		hdr.Set(cmn.HeaderObjSize, strconv.FormatInt(goi.lom.Size(), 10))
		hdr.Set(cmn.HeaderObjAtime, cmn.UnixNano2S(goi.lom.AtimeUnix()))
		if r != nil {
			hdr.Set(cmn.HeaderContentLength, strconv.FormatInt(r.Length, 10))
		} else {
			hdr.Set(cmn.HeaderContentLength, strconv.FormatInt(size, 10))
		}
	}

	w := goi.w
	if r == nil {
		reader = file
		if goi.chunked {
			// Explicitly hiding `ReadFrom` implemented for `http.ResponseWriter`
			// so the `sendfile` syscall won't be used.
			w = cmn.WriterOnly{Writer: goi.w}
			buf, slab = goi.t.gmm.Alloc(goi.lom.Size())
		}
	} else {
		buf, slab = goi.t.gmm.Alloc(r.Length)
		reader = io.NewSectionReader(file, r.Start, r.Length)
		if cksumRange {
			var cksum *cmn.CksumHash
			sgl = slab.MMSA().NewSGL(r.Length, slab.Size())
			if _, cksum, err = cmn.CopyAndChecksum(sgl, reader, buf, cksumConf.Type); err != nil {
				return
			}
			hdr.Set(cmn.HeaderObjCksumVal, cksum.Value())
			hdr.Set(cmn.HeaderObjCksumType, cksumConf.Type)
			reader = io.NewSectionReader(file, r.Start, r.Length)
		}
	}

	sent = true // At this point we mark the object as sent, regardless of the outcome.
	written, err = io.CopyBuffer(w, reader, buf)
	if err != nil {
		if cmn.IsErrConnectionReset(err) {
			return
		}
		goi.t.fsErr(err, fqn)
		goi.t.statsT.Add(stats.ErrGetCount, 1)
		return retry, sent, http.StatusInternalServerError, fmt.Errorf("failed to GET %s, err: %w", fqn, err)
	}

	// GFN: atime must be already set
	if !coldGet && !goi.isGFN {
		goi.lom.Load(false)
		goi.lom.SetAtimeUnix(goi.started.UnixNano())
		goi.lom.ReCache(true) // GFN and cold GETs already did this
	}

	// Update objects which were sent during GFN. Thanks to this we will not
	// have to resend them in rebalance. In case of race between rebalance
	// and GFN, the former wins and it will result in double send.
	if goi.isGFN {
		goi.t.rebManager.FilterAdd([]byte(goi.lom.Uname()))
	}

	delta := time.Since(goi.started)
	if glog.FastV(4, glog.SmoduleAIS) {
		s := fmt.Sprintf("GET: %s(%s), %s", goi.lom, cmn.B2S(written, 1), delta)
		if coldGet {
			s += " (cold)"
		}
		glog.Infoln(s)
	}
	goi.t.statsT.AddMany(
		stats.NamedVal64{Name: stats.GetThroughput, Value: written},
		stats.NamedVal64{Name: stats.GetLatency, Value: int64(delta)},
		stats.NamedVal64{Name: stats.GetCount, Value: 1},
	)
	return
}

///////////////////
// APPEND OBJECT //
///////////////////

func (aoi *appendObjInfo) appendObject() (newHandle string, errCode int, err error) {
	filePath := aoi.hi.filePath
	switch aoi.op {
	case cmn.AppendOp:
		var f *os.File
		if filePath == "" {
			filePath = fs.CSM.GenContentFQN(aoi.lom, fs.WorkfileType, fs.WorkfileAppend)
			f, err = aoi.lom.CreateFile(filePath)
			if err != nil {
				errCode = http.StatusInternalServerError
				return
			}
			aoi.hi.partialCksum = cmn.NewCksumHash(aoi.lom.CksumConf().Type)
		} else {
			f, err = os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, cmn.PermRWR)
			if err != nil {
				errCode = http.StatusInternalServerError
				return
			}
			cmn.Assert(aoi.hi.partialCksum != nil)
		}

		var (
			buf  []byte
			slab *memsys.Slab
		)
		if aoi.size == 0 {
			buf, slab = aoi.t.gmm.Alloc()
		} else {
			buf, slab = aoi.t.gmm.Alloc(aoi.size)
		}

		w := cmn.NewWriterMulti(f, aoi.hi.partialCksum.H)
		_, err = io.CopyBuffer(w, aoi.r, buf)

		slab.Free(buf)
		cmn.Close(f)
		if err != nil {
			errCode = http.StatusInternalServerError
			return
		}

		newHandle = combineAppendHandle(aoi.t.si.ID(), filePath, aoi.hi.partialCksum)
	case cmn.FlushOp:
		if filePath == "" {
			err = errors.New("handle not provided")
			errCode = http.StatusBadRequest
			return
		}
		cmn.Assert(aoi.hi.partialCksum != nil)
		aoi.hi.partialCksum.Finalize()
		partialCksum := aoi.hi.partialCksum.Clone()
		if aoi.cksum != nil && !partialCksum.Equal(aoi.cksum) {
			err = cmn.NewBadDataCksumError(partialCksum, aoi.cksum)
			errCode = http.StatusInternalServerError
			return
		}
		params := cluster.PromoteFileParams{
			SrcFQN:    filePath,
			Bck:       aoi.lom.Bck(),
			ObjName:   aoi.lom.ObjName,
			Cksum:     partialCksum,
			Overwrite: true,
			KeepOrig:  false,
		}
		if _, err := aoi.t.PromoteFile(params); err != nil {
			return "", 0, err
		}
	default:
		cmn.AssertMsg(false, aoi.op)
	}

	delta := time.Since(aoi.started)
	aoi.t.statsT.AddMany(
		stats.NamedVal64{Name: stats.AppendCount, Value: 1},
		stats.NamedVal64{Name: stats.AppendLatency, Value: int64(delta)},
	)
	if glog.FastV(4, glog.SmoduleAIS) {
		glog.Infof("PUT %s: %s", aoi.lom, delta)
	}
	return
}

func parseAppendHandle(handle string) (hi handleInfo, err error) {
	if handle == "" {
		return
	}
	p := strings.SplitN(handle, "|", 4)
	if len(p) != 4 {
		return hi, fmt.Errorf("invalid handle provided: %q", handle)
	}
	hi.partialCksum = cmn.NewCksumHash(p[2])
	buf, err := base64.StdEncoding.DecodeString(p[3])
	if err != nil {
		return hi, err
	}
	err = hi.partialCksum.H.(encoding.BinaryUnmarshaler).UnmarshalBinary(buf)
	if err != nil {
		return hi, err
	}
	hi.nodeID = p[0]
	hi.filePath = p[1]
	return
}

func combineAppendHandle(nodeID, filePath string, partialCksum *cmn.CksumHash) string {
	buf, err := partialCksum.H.(encoding.BinaryMarshaler).MarshalBinary()
	cmn.AssertNoErr(err)
	cksumTy := partialCksum.Type()
	cksumBinary := base64.StdEncoding.EncodeToString(buf)
	return nodeID + "|" + filePath + "|" + cksumTy + "|" + cksumBinary
}

/////////////////
// COPY OBJECT //
/////////////////

func (coi *copyObjInfo) copyObject(src *cluster.LOM, objNameTo string) (size int64, err error) {
	debug.Assert(coi.DP == nil)
	if src.Bck().IsRemote() || coi.BckTo.IsRemote() {
		// There will be no logic to create local copies etc, we can simply use copyReader
		coi.DP = &cluster.LomReader{}
		return coi.copyReader(src, objNameTo)
	}
	si := coi.t.si
	if !coi.localOnly {
		smap := coi.t.owner.smap.Get()
		if si, err = cluster.HrwTarget(coi.BckTo.MakeUname(objNameTo), smap); err != nil {
			return
		}
	}
	// remote copy
	if si.ID() != coi.t.si.ID() {
		src.Lock(false)
		if err = src.Load(false); err != nil {
			if !cmn.IsObjNotExist(err) {
				err = fmt.Errorf("%s: err: %v", src, err)
			}
			src.Unlock(false)
			return
		}
		params := cluster.SendToParams{
			ObjNameTo: objNameTo,
			Tsi:       si,
			DM:        coi.DM,
			RLocked:   true,
			NoVersion: !src.Bucket().Equal(coi.BckTo.Bck), // no versioning when buckets differ
		}
		return coi.putRemote(src, params) // r-unlocks inside
	}
	// dry-run
	if coi.DryRun {
		// TODO: replace with something similar to src.FQN == dst.FQN, but dstBck might not exist.
		if src.Bucket().Equal(coi.BckTo.Bck) && src.ObjName == objNameTo {
			return 0, nil
		}
		return src.Size(), nil
	}
	// local copy
	dst := cluster.AllocLOM(objNameTo)
	defer cluster.FreeLOM(dst)
	err = dst.Init(coi.BckTo.Bck)
	if err != nil {
		return
	}
	if src.FQN == dst.FQN { // resilvering with a single mountpath?
		return
	}

	exclusive := src.Uname() == dst.Uname()
	src.Lock(exclusive)
	defer src.Unlock(exclusive)
	if err = src.Load(false); err != nil {
		if !cmn.IsObjNotExist(err) {
			err = fmt.Errorf("%s: err: %v", src, err)
		}
		return
	}

	// unless overwriting the source w-lock the destination (see `exclusive`)
	if src.Uname() != dst.Uname() {
		dst.Lock(true)
		defer dst.Unlock(true)
		if err = dst.Load(false); err == nil {
			if src.Cksum().Equal(dst.Cksum()) {
				return
			}
		} else if cmn.IsErrBucketNought(err) {
			return
		}
	}
	dst2, err2 := src.CopyObject(dst.FQN, coi.Buf)
	if err2 == nil {
		size = src.Size()
		if coi.finalize {
			coi.t.putMirror(dst2)
		}
	}
	err = err2
	cluster.FreeLOM(dst2)
	return
}

/////////////////
// COPY READER //
/////////////////

// copyReader puts a new object to a cluster, according to a reader taken from coi.DP.Reader(lom) The reader returned
// from coi.DP is responsible for any locking or source LOM, if necessary. If the reader doesn't take any locks, it has
// to consider object content changing in the middle of copying.
//
// LOM can be meta of a cloud object. It creates some problems. However, it's DP who is responsible for providing a reader,
// so DP should tak any steps necessary to do so. It includes handling cold get, warm get etc.
//
// If destination bucket is remote bucket, copyReader will always create a cached copy of an object on one of the
// targets as well as make put to the relevant backend provider.
// TODO: Make it possible to skip caching an object from a cloud bucket.
func (coi *copyObjInfo) copyReader(lom *cluster.LOM, objNameTo string) (size int64, err error) {
	var (
		reader  cmn.ReadOpenCloser
		cleanUp func()
		si      = coi.t.si
	)
	debug.Assert(coi.DP != nil)
	if si, err = cluster.HrwTarget(coi.BckTo.MakeUname(objNameTo), coi.t.owner.smap.Get()); err != nil {
		return
	}

	if err = lom.Load(); err != nil {
		return
	}

	if si.ID() != coi.t.si.ID() {
		params := cluster.SendToParams{
			ObjNameTo: objNameTo,
			Tsi:       si,
			DM:        coi.DM,
			NoVersion: !lom.Bucket().Equal(coi.BckTo.Bck), // no versioning when buckets differ
		}
		return coi.putRemote(lom, params)
	}

	// DryRun: just get a reader and discard it. Init on dstLOM would cause and error as dstBck doesn't exist.
	if coi.DryRun {
		return coi.dryRunCopyReader(lom)
	}

	dst := cluster.AllocLOM(objNameTo)
	defer cluster.FreeLOM(dst)
	if err = dst.Init(coi.BckTo.Bck); err != nil {
		return
	}
	if lom.Bucket().Equal(coi.BckTo.Bck) {
		dst.SetVersion(lom.Version())
	}

	if reader, _, cleanUp, err = coi.DP.Reader(lom); err != nil {
		return 0, err
	}
	defer cleanUp()

	// Set the correct recvType: some transactions must update the object
	// in the Cloud(if destination is a Cloud bucket).
	recvType := cluster.Migrated
	if coi.DM != nil {
		recvType = coi.DM.RecvType()
	}
	params := cluster.PutObjectParams{
		Tag:      "copy-dp",
		Reader:   reader,
		RecvType: recvType,
	}
	if err := coi.t.PutObject(dst, params); err != nil {
		return 0, err
	}

	return lom.Size(), nil
}

func (coi *copyObjInfo) dryRunCopyReader(lom *cluster.LOM) (size int64, err error) {
	cmn.Assert(coi.DryRun)
	cmn.Assert(coi.DP != nil)

	var (
		reader  io.ReadCloser
		cleanUp func()
	)

	if reader, _, cleanUp, err = coi.DP.Reader(lom); err != nil {
		return 0, err
	}

	defer func() {
		reader.Close()
		cleanUp()
	}()

	return io.Copy(ioutil.Discard, reader)
}

// PUT object onto designated target
func (coi *copyObjInfo) putRemote(lom *cluster.LOM, params cluster.SendToParams) (size int64, err error) {
	if coi.DP == nil {
		var file *cmn.FileHandle
		if coi.DryRun {
			if params.RLocked {
				defer lom.Unlock(false)
			}
			return lom.Size(), nil
		}
		if file, err = cmn.NewFileHandle(lom.FQN); err != nil {
			return 0, fmt.Errorf("failed to open %s: %w", lom.FQN, err)
		}
		fi, err := file.Stat()
		if err != nil {
			return 0, fmt.Errorf("failed to stat %s: %w", lom.FQN, err)
		}
		params.Reader = file
		params.HdrMeta = lom
		size = fi.Size()
	} else {
		var cleanUp func()
		if params.Reader, params.HdrMeta, cleanUp, err = coi.DP.Reader(lom); err != nil {
			return
		}
		defer cleanUp()
		if coi.DryRun {
			size, err = io.Copy(ioutil.Discard, params.Reader)
			cmn.Close(params.Reader)
			return
		}
		// NOTE: return the current size as resulting (transformed) size may not be known.
		size = lom.Size()
	}
	debug.Assert(params.HdrMeta != nil)
	if params.NoVersion {
		params.HdrMeta = cmn.NewHdrMetaCustomVersion(params.HdrMeta, "")
	}
	params.BckTo = coi.BckTo
	// either stream
	if params.DM != nil {
		err = _sendObjDM(lom.Clone(lom.FQN) /*free in the callback below*/, params)
		return
	}
	// or PUT
	err = coi.t._sendPUT(lom, params)
	if params.RLocked {
		lom.Unlock(false)
	}
	return
}

// streaming send via bundle.DataMover
func _sendObjDM(lom *cluster.LOM, params cluster.SendToParams) (err error) {
	o := transport.AllocSend()
	o.Hdr.FromHdrProvider(params.HdrMeta, params.ObjNameTo, params.BckTo.Bck, nil)
	o.CmplPtr = unsafe.Pointer(lom)
	o.Callback = func(_ transport.ObjHdr, _ io.ReadCloser, lomptr unsafe.Pointer, _ error) {
		_freeLptr(lomptr, params.RLocked)
	}
	err = params.DM.Send(o, params.Reader, params.Tsi)
	if err != nil {
		_freeLptr(unsafe.Pointer(lom), params.RLocked)
	}
	return
}

func _freeLptr(lomptr unsafe.Pointer, locked bool) {
	lom := (*cluster.LOM)(lomptr)
	if locked {
		lom.Unlock(false)
	}
	cluster.FreeLOM(lom)
}

///////////////
// mem pools //
///////////////

var (
	goiPool, poiPool, coiPool sync.Pool

	goi0 getObjInfo
	poi0 putObjInfo
	coi0 copyObjInfo
)

func allocGetObjInfo() (a *getObjInfo) {
	if v := goiPool.Get(); v != nil {
		a = v.(*getObjInfo)
		return
	}
	return &getObjInfo{}
}

func freeGetObjInfo(a *getObjInfo) {
	*a = goi0
	goiPool.Put(a)
}

func allocPutObjInfo() (a *putObjInfo) {
	if v := poiPool.Get(); v != nil {
		a = v.(*putObjInfo)
		return
	}
	return &putObjInfo{}
}

func freePutObjInfo(a *putObjInfo) {
	*a = poi0
	poiPool.Put(a)
}

func allocCopyObjInfo() (a *copyObjInfo) {
	if v := coiPool.Get(); v != nil {
		a = v.(*copyObjInfo)
		return
	}
	return &copyObjInfo{}
}

func freeCopyObjInfo(a *copyObjInfo) {
	*a = coi0
	coiPool.Put(a)
}

extension: go


text:
package pipeline

import (
	"github.com/jin06/binlogo/pkg/store/dao/dao_node"
	"github.com/jin06/binlogo/pkg/store/dao/dao_pipe"
	"github.com/jin06/binlogo/pkg/store/dao/dao_sche"
	"github.com/jin06/binlogo/pkg/store/model/node"
	"github.com/jin06/binlogo/pkg/store/model/pipeline"
	"github.com/jin06/binlogo/pkg/store/model/scheduler"
)

// Item for front display
type Item struct {
	Pipeline *pipeline.Pipeline `json:"pipeline"`
	Info     *Info              `json:"info"`
}

// Info for front display
type Info struct {
	BindNode *node.Node         `json:"bind_node"`
	Instance *pipeline.Instance `json:"instance"`
}

// CompleteInfo complete info for item
func CompleteInfo(i *Item) (err error) {
	err = CompletePipelineBind(i, nil)
	if err != nil {
		return
	}
	err = completePipelineRun(i, nil)
	if err != nil {
		return
	}
	return
}

func completePipelineRun(i *Item, pMap map[string]*pipeline.Instance) (err error) {
	if i == nil {
		return
	}
	if i.Pipeline == nil {
		return
	}
	i.Info.Instance = &pipeline.Instance{}
	if pMap != nil {
		if _, ok := pMap[i.Pipeline.Name]; ok {
			i.Info.Instance = pMap[i.Pipeline.Name]
		}
		return
	}

	ins, err := dao_pipe.GetInstance(i.Pipeline.Name)
	if err != nil {
		return
	}
	i.Info.Instance = ins
	return
}

// CompletePipelineBind complete pipeline bind info
func CompletePipelineBind(i *Item, pb *scheduler.PipelineBind) (err error) {
	if pb == nil {
		pb, err = dao_sche.GetPipelineBind()
		if err != nil {
			return
		}
	}
	var nodeName string
	for k, v := range pb.Bindings {
		if k == i.Pipeline.Name {
			nodeName = v
			break
		}
	}
	if i.Info == nil {
		i.Info = &Info{}
	}
	if nodeName == "" {
		i.Info.BindNode = &node.Node{}
		return
	}
	n, err := dao_node.GetNode(nodeName)
	if err != nil {
		return
	}
	i.Info.BindNode = n
	return
}

func completeEmpty(i *Item) {
	sender := i.Pipeline.Output.Sender
	if sender.RocketMQ == nil {
		sender.RocketMQ = &pipeline.RocketMQ{}
	}
	if sender.Redis == nil {
		sender.Redis = &pipeline.Redis{}
	}
	if sender.Kafka == nil {
		sender.Kafka = &pipeline.Kafka{}
	}
	if sender.Http == nil {
		sender.Http = &pipeline.Http{}
	}
}

// CompleteInfoList complete pipeline list info
func CompleteInfoList(list []*Item) (err error) {
	pb, err := dao_sche.GetPipelineBind()
	if err != nil {
		return
	}
	allInstance, err := dao_pipe.AllInstanceMap()
	if err != nil {
		return
	}
	for _, v := range list {
		if err1 := CompletePipelineBind(v, pb); err1 != nil {
			continue
		}
		if err2 := completePipelineRun(v, allInstance); err2 != nil {
			return
		}
		completeEmpty(v)
	}
	return
}

license: MIT

license_type: permissive

branch_name: refs/heads/master

gha_license_id: MIT

gha_language: Go


text:
Automatically generated by Mendeley Desktop 1.19.4
Any changes to this file will be lost if it is regenerated by Mendeley.

BibTeX export options can be customized via Preferences -> BibTeX in Mendeley Desktop

@misc{Russell:2003:AIM:773294,
abstract = {In recent years, deep artificial neural networks (including recurrent ones) have won numerous contests in pattern recognition and machine learning. This historical survey compactly summarizes relevant work, much of it from the previous millennium. Shallow and Deep Learners are distinguished by the depth of their credit assignment paths, which are chains of possibly learnable, causal links between actions and effects. I review deep supervised learning (also recapitulating the history of backpropagation), unsupervised learning, reinforcement learning {\&} evolutionary computation, and indirect search for short programs encoding deep and large networks.},
archivePrefix = {arXiv},
arxivId = {1404.7828},
author = {Schmidhuber, J{\"{u}}rgen},
booktitle = {Neural Networks},
doi = {10.1016/j.neunet.2014.09.003},
edition = {2},
eprint = {1404.7828},
isbn = {0893-6080},
issn = {18792782},
keywords = {Deep learning,Evolutionary computation,Reinforcement learning,Supervised learning,Unsupervised learning},
month = {jan},
pages = {85--117},
pmid = {25462637},
publisher = {Pearson Education},
title = {{Deep Learning in neural networks: An overview}},
url = {https://www.sciencedirect.com/science/article/pii/S0893608014002135?via{\%}3Dihub},
volume = {61},
year = {2015}
}

license: Unlicense

repository: sepmein/references

repo_name: sepmein/references

extension: bib


text:
/**
	@author
	Ryan Pavlik ( <rpavlik@iastate.edu> http://academic.cleardefinition.com/ ),
	Iowa State University
	Virtual Reality Applications Center and
	Human-Computer Interaction Graduate Program
*/

#define BOOST_TEST_MODULE RunLoopManagerBoost

#include <util/RunLoopManagerBoost.h>

typedef util::RunLoopManagerBoost RunLoopManager;

#include "RunLoopManager_common.h"

repository: rpavlik/util-headers

repo_name: rpavlik/util-headers

branch_name: refs/heads/main

extension: cpp


text:
/*
Package cmd - ...

Copyright © 2021 Sam Mingolelli <github@lamolabs.org>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package cmd

import (
	"context"
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/go-ping/ping"
	"github.com/gookit/color"
	"github.com/jedib0t/go-pretty/v6/table"
	gobrex "github.com/kujtimiihoxha/go-brace-expansion"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// svrsCmd represents the svrs command
var svrsCmd = &cobra.Command{
	Use:   "svrs",
	Short: "Run diagnostics verifying connectivity to well known servers thru a VPN connection",
	Long: `
doxctl's 'svrs' subcommand can help triage & test connectivity to 'well known servers'
thru a VPN connection to servers which have been defined in your '.doxctl.yaml'
configuration file.
	`,
	PreRun: func(cmd *cobra.Command, args []string) {
		// Process config, environment variables, and flags
		viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
		viper.AutomaticEnv()

		// Populate as much info as we can from viper
		err := viper.Unmarshal(&conf)
		if err != nil {
			fmt.Printf("could not retrieve supplied project settings: %s\n", err)
			os.Exit(1)
		}
	},
	Run: svrsExecute,
}

var svrsReachableChk bool

func init() {
	rootCmd.AddCommand(svrsCmd)

	svrsCmd.Flags().BoolVarP(&svrsReachableChk, "svrsReachableChk", "s", false, "Check if well known servers are reachable")
	svrsCmd.Flags().BoolVarP(&allChk, "allChk", "a", false, "Run all the checks in this subcommand module")
}

func svrsExecute(cmd *cobra.Command, args []string) {
	switch {
	case svrsReachableChk:
		svrsReachChk()
	case allChk:
		svrsReachChk()
	default:
		cmd.Usage()
		fmt.Printf("\n\n\n")
		os.Exit(1)
	}
}

// Check if well known servers are pingable & reachable
func svrsReachChk() {
	color.Info.Tips("Attempting to ping all well known servers, this may take a few...\n")

	// Table head
	t := table.NewWriter()
	t.SetTitle("Well known Servers Reachable Checks")
	t.SetOutputMirror(os.Stdout)
	t.SetStyle(table.StyleLight)
	t.SetColumnConfigs([]table.ColumnConfig{
		{Number: 1, WidthMin: 40},
		{Number: 2, WidthMin: 20},
	})
	t.AppendHeader(table.Row{"Host", "Service", "Reachable?", "Ping Performance"})

	/* Walk through list of hosts, attempt to ping 'em.
	 *
	 * 1 - Loop through the list of svcs in .doxctl.yaml file
	 * 2 - Expand brace definitions of hosts determining all the `permutations`
	 * 3 - Go through perms. attempt to ping each and confirm that it was reached
	 * 4 - Confirm response packet was received (PacketLoss & PacketRecv)
	 * 5 - If more than FailThreshold occurs for either Packet* stop trying, call the rest failed
	 *
	 */
	pingFailures := 0
	reachFailures := 0

	for _, i := range conf.Svcs {
		fmt.Printf("   --- Working through svc: %s\n", i.Svc)

		for _, j := range i.Svrs {
			permutations := gobrex.Expand(j)

			for _, permutation := range permutations {

				// Attempt to resolve hostname prior to ping
				ctx, cancel := context.WithTimeout(context.TODO(), (conf.DNSLookupTimeout * time.Millisecond))
				defer cancel() // important to avoid a resource leak
				var r net.Resolver
				ip, err := r.LookupHost(ctx, permutation)

				if err != nil || len(ip) == 0 {
					t.AppendRow([]interface{}{permutation, i.Svc, false, "N/A"})
					continue
				}

				// Attempt to ping each host, any that fail keep a tally
				pinger, err := ping.NewPinger(permutation)
				if err != nil {
					t.AppendRow([]interface{}{permutation, i.Svc, false, "N/A"})
					pingFailures++
					continue
				}

				pinger.Timeout = conf.PingTimeout * time.Millisecond
				pOut := make(chan *ping.Statistics)
				go func(p *ping.Pinger, s chan *ping.Statistics) {
					p.Run()
					s <- p.Statistics()
				}(pinger, pOut)
				stats := <-pOut
				pingPerf := fmt.Sprintf("rnd-trp avg = %v", stats.AvgRtt)

				// Tally fails due to failed/missing responses
				packetAck := (stats.PacketLoss == 0 && stats.PacketsRecv > 0)
				if !packetAck {
					reachFailures++
				}

				t.AppendRow([]interface{}{permutation, i.Svc, packetAck, pingPerf})

			}
		}
		t.AppendSeparator()
	}
	fmt.Printf("\n\n   ...one sec, preparing `ping` results...\n\n")

	if pingFailures > conf.FailThreshold || reachFailures > conf.FailThreshold {
		fmt.Println("")
		color.Warn.Tips("More than %d hosts appear to be unreachable....\n\n", conf.FailThreshold)
	}

	time.Sleep(4 * time.Second)

	t.AppendSeparator()
	t.Render()

	if pingFailures > 0 || reachFailures > 0 {
		fmt.Println("")
		color.Warn.Tips(`

   Your VPN client does not appear to be functioning properly, it's likely one or more of the following:

      - Well known servers are unreachable via ping   --- try running 'doxctl vpn -h'
      - Servers are unresovlable in DNS               --- try running 'doxctl dns -h'
      - VPN client is otherwise misconfigured!
	`)
	}
}

repository: ecmingol/doxctl

repo_name: ecmingol/doxctl


text:
set(RASPBERRY_DIR_BASE /usr/local/toolchains/armv6-logicals-linux-gnueabihf)
set(RASPBERRY_DIR_COMPILER ${RASPBERRY_DIR_BASE}/bin)

set(CMAKE_C_COMPILER ${RASPBERRY_DIR_COMPILER}/armv6-logicals-linux-gnueabihf-gcc)
set(CMAKE_CXX_COMPILER ${RASPBERRY_DIR_COMPILER}/armv6-logicals-linux-gnueabihf-g++)
set(RTS_PACKAGE_EXTRA_LINUX DEB)

# Name of the target platform
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_PROCESSOR arm)

# compiler flags
set(CMAKE_C_FLAGS "--std=gnu99 -Werror=implicit-function-declaration" CACHE STRING "" FORCE)

set(CMAKE_CXX_CREATE_SHARED_LIBRARY "<CMAKE_CXX_COMPILER> <CMAKE_SHARED_LIBRARY_CXX_FLAGS> <LANGUAGE_COMPILE_FLAGS> <LINK_FLAGS> <CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS> <SONAME_FLAG><TARGET_SONAME> -o <TARGET> -Wl,--start-group <OBJECTS> <LINK_LIBRARIES> -Wl,--end-group")
set(CMAKE_CXX_LINK_EXECUTABLE "<CMAKE_CXX_COMPILER> <FLAGS> <CMAKE_CXX_LINK_FLAGS> <LINK_FLAGS> -Wl,--start-group <OBJECTS> <LINK_LIBRARIES> -Wl,--end-group -o <TARGET>")

repository: rpoisel/LoRa-RaspberryPi

repo_name: rpoisel/LoRa-RaspberryPi

extension: cmake


text:
SUMMARY = "A zero-dependency DBus library for Python with asyncio support"
HOMEPAGE = "https://github.com/acrisci/python-dbus-next"
LICENSE = "MIT"
LIC_FILES_CHKSUM = "file://LICENSE;md5=94e750c96e56788499b56c81de91431c"

SRC_URI[sha256sum] = "f656a3d3450b670f228248ffb1c3a703a69c4a8cb10cce63b108f17c8bd6c3de"

PYPI_PACKAGE = "dbus_next"

inherit pypi setuptools3

BBCLASSEXTEND = "native nativesdk"

repository: ni/meta-openembedded

repo_name: ni/meta-openembedded

branch_name: refs/heads/nilrt/master/hardknott

gha_language: BitBake

extension: bb


text:
/*
 * This code was generated by
 * ___ _ _ _ _ _    _ ____    ____ ____ _    ____ ____ _  _ ____ ____ ____ ___ __   __
 *  |  | | | | |    | |  | __ |  | |__| | __ | __ |___ |\ | |___ |__/ |__|  | |  | |__/
 *  |  |_|_| | |___ | |__|    |__| |  | |    |__] |___ | \| |___ |  \ |  |  | |__| |  \
 *
 * Twilio - Api
 * This is the public Twilio REST API.
 *
 * NOTE: This class is auto generated by OpenAPI Generator.
 * https://openapi-generator.tech
 * Do not edit the class manually.
 */

package openapi

import (
	"encoding/json"
	"fmt"
	"net/url"
	"strings"

	"github.com/twilio/twilio-go/client"
)

// Optional parameters for the method 'FetchConference'
type FetchConferenceParams struct {
	// The SID of the [Account](https://www.twilio.com/docs/iam/api/account) that created the Conference resource(s) to fetch.
	PathAccountSid *string `json:"PathAccountSid,omitempty"`
}

func (params *FetchConferenceParams) SetPathAccountSid(PathAccountSid string) *FetchConferenceParams {
	params.PathAccountSid = &PathAccountSid
	return params
}

// Fetch an instance of a conference
func (c *ApiService) FetchConference(Sid string, params *FetchConferenceParams) (*ApiV2010Conference, error) {
	path := "/2010-04-01/Accounts/{AccountSid}/Conferences/{Sid}.json"
	if params != nil && params.PathAccountSid != nil {
		path = strings.Replace(path, "{"+"AccountSid"+"}", *params.PathAccountSid, -1)
	} else {
		path = strings.Replace(path, "{"+"AccountSid"+"}", c.requestHandler.Client.AccountSid(), -1)
	}
	path = strings.Replace(path, "{"+"Sid"+"}", Sid, -1)

	data := url.Values{}
	headers := make(map[string]interface{})

	resp, err := c.requestHandler.Get(c.baseURL+path, data, headers)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	ps := &ApiV2010Conference{}
	if err := json.NewDecoder(resp.Body).Decode(ps); err != nil {
		return nil, err
	}

	return ps, err
}

// Optional parameters for the method 'ListConference'
type ListConferenceParams struct {
	// The SID of the [Account](https://www.twilio.com/docs/iam/api/account) that created the Conference resource(s) to read.
	PathAccountSid *string `json:"PathAccountSid,omitempty"`
	// The `date_created` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that started on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify  conferences that started on or after midnight on a date, use `>=YYYY-MM-DD`.
	DateCreated *string `json:"DateCreated,omitempty"`
	// The `date_created` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that started on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify  conferences that started on or after midnight on a date, use `>=YYYY-MM-DD`.
	DateCreatedBefore *string `json:"DateCreated<,omitempty"`
	// The `date_created` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that started on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify  conferences that started on or after midnight on a date, use `>=YYYY-MM-DD`.
	DateCreatedAfter *string `json:"DateCreated>,omitempty"`
	// The `date_updated` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that were last updated on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify conferences that were last updated on or after midnight on a given date, use  `>=YYYY-MM-DD`.
	DateUpdated *string `json:"DateUpdated,omitempty"`
	// The `date_updated` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that were last updated on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify conferences that were last updated on or after midnight on a given date, use  `>=YYYY-MM-DD`.
	DateUpdatedBefore *string `json:"DateUpdated<,omitempty"`
	// The `date_updated` value, specified as `YYYY-MM-DD`, of the resources to read. To read conferences that were last updated on or before midnight on a date, use `<=YYYY-MM-DD`, and to specify conferences that were last updated on or after midnight on a given date, use  `>=YYYY-MM-DD`.
	DateUpdatedAfter *string `json:"DateUpdated>,omitempty"`
	// The string that identifies the Conference resources to read.
	FriendlyName *string `json:"FriendlyName,omitempty"`
	// The status of the resources to read. Can be: `init`, `in-progress`, or `completed`.
	Status *string `json:"Status,omitempty"`
	// How many resources to return in each list page. The default is 50, and the maximum is 1000.
	PageSize *int `json:"PageSize,omitempty"`
	// Max number of records to return.
	Limit *int `json:"limit,omitempty"`
}

func (params *ListConferenceParams) SetPathAccountSid(PathAccountSid string) *ListConferenceParams {
	params.PathAccountSid = &PathAccountSid
	return params
}
func (params *ListConferenceParams) SetDateCreated(DateCreated string) *ListConferenceParams {
	params.DateCreated = &DateCreated
	return params
}
func (params *ListConferenceParams) SetDateCreatedBefore(DateCreatedBefore string) *ListConferenceParams {
	params.DateCreatedBefore = &DateCreatedBefore
	return params
}
func (params *ListConferenceParams) SetDateCreatedAfter(DateCreatedAfter string) *ListConferenceParams {
	params.DateCreatedAfter = &DateCreatedAfter
	return params
}
func (params *ListConferenceParams) SetDateUpdated(DateUpdated string) *ListConferenceParams {
	params.DateUpdated = &DateUpdated
	return params
}
func (params *ListConferenceParams) SetDateUpdatedBefore(DateUpdatedBefore string) *ListConferenceParams {
	params.DateUpdatedBefore = &DateUpdatedBefore
	return params
}
func (params *ListConferenceParams) SetDateUpdatedAfter(DateUpdatedAfter string) *ListConferenceParams {
	params.DateUpdatedAfter = &DateUpdatedAfter
	return params
}
func (params *ListConferenceParams) SetFriendlyName(FriendlyName string) *ListConferenceParams {
	params.FriendlyName = &FriendlyName
	return params
}
func (params *ListConferenceParams) SetStatus(Status string) *ListConferenceParams {
	params.Status = &Status
	return params
}
func (params *ListConferenceParams) SetPageSize(PageSize int) *ListConferenceParams {
	params.PageSize = &PageSize
	return params
}
func (params *ListConferenceParams) SetLimit(Limit int) *ListConferenceParams {
	params.Limit = &Limit
	return params
}

// Retrieve a single page of Conference records from the API. Request is executed immediately.
func (c *ApiService) PageConference(params *ListConferenceParams, pageToken, pageNumber string) (*ListConferenceResponse, error) {
	path := "/2010-04-01/Accounts/{AccountSid}/Conferences.json"

	if params != nil && params.PathAccountSid != nil {
		path = strings.Replace(path, "{"+"AccountSid"+"}", *params.PathAccountSid, -1)
	} else {
		path = strings.Replace(path, "{"+"AccountSid"+"}", c.requestHandler.Client.AccountSid(), -1)
	}

	data := url.Values{}
	headers := make(map[string]interface{})

	if params != nil && params.DateCreated != nil {
		data.Set("DateCreated", fmt.Sprint(*params.DateCreated))
	}
	if params != nil && params.DateCreatedBefore != nil {
		data.Set("DateCreated<", fmt.Sprint(*params.DateCreatedBefore))
	}
	if params != nil && params.DateCreatedAfter != nil {
		data.Set("DateCreated>", fmt.Sprint(*params.DateCreatedAfter))
	}
	if params != nil && params.DateUpdated != nil {
		data.Set("DateUpdated", fmt.Sprint(*params.DateUpdated))
	}
	if params != nil && params.DateUpdatedBefore != nil {
		data.Set("DateUpdated<", fmt.Sprint(*params.DateUpdatedBefore))
	}
	if params != nil && params.DateUpdatedAfter != nil {
		data.Set("DateUpdated>", fmt.Sprint(*params.DateUpdatedAfter))
	}
	if params != nil && params.FriendlyName != nil {
		data.Set("FriendlyName", *params.FriendlyName)
	}
	if params != nil && params.Status != nil {
		data.Set("Status", *params.Status)
	}
	if params != nil && params.PageSize != nil {
		data.Set("PageSize", fmt.Sprint(*params.PageSize))
	}

	if pageToken != "" {
		data.Set("PageToken", pageToken)
	}
	if pageNumber != "" {
		data.Set("Page", pageNumber)
	}

	resp, err := c.requestHandler.Get(c.baseURL+path, data, headers)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	ps := &ListConferenceResponse{}
	if err := json.NewDecoder(resp.Body).Decode(ps); err != nil {
		return nil, err
	}

	return ps, err
}

// Lists Conference records from the API as a list. Unlike stream, this operation is eager and loads 'limit' records into memory before returning.
func (c *ApiService) ListConference(params *ListConferenceParams) ([]ApiV2010Conference, error) {
	response, errors := c.StreamConference(params)

	records := make([]ApiV2010Conference, 0)
	for record := range response {
		records = append(records, record)
	}

	if err := <-errors; err != nil {
		return nil, err
	}

	return records, nil
}

// Streams Conference records from the API as a channel stream. This operation lazily loads records as efficiently as possible until the limit is reached.
func (c *ApiService) StreamConference(params *ListConferenceParams) (chan ApiV2010Conference, chan error) {
	if params == nil {
		params = &ListConferenceParams{}
	}
	params.SetPageSize(client.ReadLimits(params.PageSize, params.Limit))

	recordChannel := make(chan ApiV2010Conference, 1)
	errorChannel := make(chan error, 1)

	response, err := c.PageConference(params, "", "")
	if err != nil {
		errorChannel <- err
		close(recordChannel)
		close(errorChannel)
	} else {
		go c.streamConference(response, params, recordChannel, errorChannel)
	}

	return recordChannel, errorChannel
}

func (c *ApiService) streamConference(response *ListConferenceResponse, params *ListConferenceParams, recordChannel chan ApiV2010Conference, errorChannel chan error) {
	curRecord := 1

	for response != nil {
		responseRecords := response.Conferences
		for item := range responseRecords {
			recordChannel <- responseRecords[item]
			curRecord += 1
			if params.Limit != nil && *params.Limit < curRecord {
				close(recordChannel)
				close(errorChannel)
				return
			}
		}

		record, err := client.GetNext(c.baseURL, response, c.getNextListConferenceResponse)
		if err != nil {
			errorChannel <- err
			break
		} else if record == nil {
			break
		}

		response = record.(*ListConferenceResponse)
	}

	close(recordChannel)
	close(errorChannel)
}

func (c *ApiService) getNextListConferenceResponse(nextPageUrl string) (interface{}, error) {
	if nextPageUrl == "" {
		return nil, nil
	}
	resp, err := c.requestHandler.Get(nextPageUrl, nil, nil)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	ps := &ListConferenceResponse{}
	if err := json.NewDecoder(resp.Body).Decode(ps); err != nil {
		return nil, err
	}
	return ps, nil
}

// Optional parameters for the method 'UpdateConference'
type UpdateConferenceParams struct {
	// The SID of the [Account](https://www.twilio.com/docs/iam/api/account) that created the Conference resource(s) to update.
	PathAccountSid *string `json:"PathAccountSid,omitempty"`
	//
	Status *string `json:"Status,omitempty"`
	// The URL we should call to announce something into the conference. The URL may return an MP3 file, a WAV file, or a TwiML document that contains `<Play>`, `<Say>`, `<Pause>`, or `<Redirect>` verbs.
	AnnounceUrl *string `json:"AnnounceUrl,omitempty"`
	// The HTTP method used to call `announce_url`. Can be: `GET` or `POST` and the default is `POST`
	AnnounceMethod *string `json:"AnnounceMethod,omitempty"`
}

func (params *UpdateConferenceParams) SetPathAccountSid(PathAccountSid string) *UpdateConferenceParams {
	params.PathAccountSid = &PathAccountSid
	return params
}
func (params *UpdateConferenceParams) SetStatus(Status string) *UpdateConferenceParams {
	params.Status = &Status
	return params
}
func (params *UpdateConferenceParams) SetAnnounceUrl(AnnounceUrl string) *UpdateConferenceParams {
	params.AnnounceUrl = &AnnounceUrl
	return params
}
func (params *UpdateConferenceParams) SetAnnounceMethod(AnnounceMethod string) *UpdateConferenceParams {
	params.AnnounceMethod = &AnnounceMethod
	return params
}

//
func (c *ApiService) UpdateConference(Sid string, params *UpdateConferenceParams) (*ApiV2010Conference, error) {
	path := "/2010-04-01/Accounts/{AccountSid}/Conferences/{Sid}.json"
	if params != nil && params.PathAccountSid != nil {
		path = strings.Replace(path, "{"+"AccountSid"+"}", *params.PathAccountSid, -1)
	} else {
		path = strings.Replace(path, "{"+"AccountSid"+"}", c.requestHandler.Client.AccountSid(), -1)
	}
	path = strings.Replace(path, "{"+"Sid"+"}", Sid, -1)

	data := url.Values{}
	headers := make(map[string]interface{})

	if params != nil && params.Status != nil {
		data.Set("Status", *params.Status)
	}
	if params != nil && params.AnnounceUrl != nil {
		data.Set("AnnounceUrl", *params.AnnounceUrl)
	}
	if params != nil && params.AnnounceMethod != nil {
		data.Set("AnnounceMethod", *params.AnnounceMethod)
	}

	resp, err := c.requestHandler.Post(c.baseURL+path, data, headers)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	ps := &ApiV2010Conference{}
	if err := json.NewDecoder(resp.Body).Decode(ps); err != nil {
		return nil, err
	}

	return ps, err
}

repository: twilio/twilio-go

repo_name: twilio/twilio-go


text:
package main

import (
	"os"

	"github.com/ArmandGrillet/mesos-cli/cmd"
	"github.com/dcos/dcos-cli/api"
	"github.com/dcos/dcos-cli/pkg/cli"
)

func main() {
	ctx := cli.NewContext(cli.NewOsEnvironment())
	if err := run(ctx, os.Args); err != nil {
		os.Exit(1)
	}
}

func run(ctx api.Context, args []string) error {
	return cmd.NewMesosCommand(ctx).Execute()
}

repository: armandgrillet/mesos-cli

repo_name: armandgrillet/mesos-cli


text:
//
// Copyright © 2019 Stephen Hoekstra
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package cmd

import (
	"fmt"
	"os"

	"github.com/shoekstra/repo-settings/internal/config"
	"github.com/shoekstra/repo-settings/internal/gitlab"
	"github.com/spf13/cobra"
)

var dryRun bool
var gitlabToken string
var gitlabURL string
var cfgFile string

// NewRepoDefaultsCmd represents the command
func NewCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "repo-settings",
		Short: "CLI to configure repository settings in a GitLab group or project.",
		Long: `
A simple CLI to configure repositories settings across various repository
hosters.

It reads a configuration file containing a GitHub organisation (not yet
supported) and/or GitLab group and will configure all repositories
found within with the defined settings.

To get started, create a configuration file and pass the --config option.`,
		Run: func(cmd *cobra.Command, args []string) {
			if err := runCmd(); err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
		},
	}

	// Add some flags.
	cmd.Flags().BoolVarP(&dryRun, "dry-drun", "d", false, "perform a dry run")
	cmd.Flags().StringVar(&gitlabToken, "gitlab-token", "", "GitLab API token")
	cmd.Flags().StringVar(&gitlabURL, "gitlab-url", "", "GitLab API URL")
	cmd.Flags().StringVarP(&cfgFile, "config", "c", "", "path to config file")

	return cmd
}

func runCmd() error {
	if err := validateCfgFile(); err != nil {
		return err
	}

	cfg, err := config.Load(cfgFile)
	if err != nil {
		return err
	}

	if cfg.GitLab != nil {
		if err := cfg.GitLab.LoadCreds(gitlabToken, gitlabURL); err != nil {
			return err
		}

		if dryRun {
			cfg.GitLab.DryRun = true
		}

		// Update projects in configured groups.
		if cfg.GitLab.Groups != nil {
			if err := gitlab.UpdateProjectsInGroups(cfg.GitLab); err != nil {
				return err
			}
		}
	}

	return nil
}

func validateCfgFile() error {
	// Print help if no config file is passed
	if cfgFile == "" {
		cmd := NewCmd()
		cmd.Help()
		os.Exit(0)
	}

	// Test config file exists
	if _, err := os.Stat(cfgFile); err != nil {
		if os.IsNotExist(err) {
			return fmt.Errorf("Cannot open config file %s", cfgFile)
		}
	}

	return nil
}

repository: shoekstra/repo-settings

repo_name: shoekstra/repo-settings


text:
require xorg-lib-common.inc

DESCRIPTION = "	Touchscreen calibration client library"
LICENSE = "BSD-X"
DEPENDS = "virtual/libx11 calibrateproto libxext"
PV = "0.0+gitr${SRCREV}"
PE = "1"

SRC_URI = "git://anongit.freedesktop.org/git/xorg/lib/libXCalibrate;protocol=git"

S = "${WORKDIR}/git"

FILES_${PN}-locale += "${datadir}/X11/locale"


text:
@misc{reddy2020systems,
  title={Systems and methods for deep-learning based super-resolution using multiple degradations on-demand learning},
  author={Reddy, Sai Narsi Reddy Donthi and Dai, Qiqin},
  year={2020},
  month=jul # "~30",
  publisher={Google Patents},
  note={US Patent App. 16/256,371},
  url={https://patents.google.com/patent/US20200242515A1/en}

}

@misc{reddy2020methods,
  title={Methods for deep-learning based super-resolution using high-frequency loss},
  author={Reddy, Sai Narsi Reddy Donthi},
  year={2020},
  month=jul # "~16",
  publisher={Google Patents},
  note={US Patent App. 16/247,450},
  url={https://patents.google.com/patent/US20200226718A1/en}
}


text:
// Code generated by go-swagger; DO NOT EDIT.

package dashboards_v1

// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command

import (
	"fmt"
	"io"

	"github.com/go-openapi/runtime"
	"github.com/go-openapi/strfmt"

	"github.com/polyaxon/sdks/go/http_client/v1/service_model"
)

// GetDashboardReader is a Reader for the GetDashboard structure.
type GetDashboardReader struct {
	formats strfmt.Registry
}

// ReadResponse reads a server response into the received o.
func (o *GetDashboardReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
	switch response.Code() {
	case 200:
		result := NewGetDashboardOK()
		if err := result.readResponse(response, consumer, o.formats); err != nil {
			return nil, err
		}
		return result, nil
	case 204:
		result := NewGetDashboardNoContent()
		if err := result.readResponse(response, consumer, o.formats); err != nil {
			return nil, err
		}
		return result, nil
	case 403:
		result := NewGetDashboardForbidden()
		if err := result.readResponse(response, consumer, o.formats); err != nil {
			return nil, err
		}
		return nil, result
	case 404:
		result := NewGetDashboardNotFound()
		if err := result.readResponse(response, consumer, o.formats); err != nil {
			return nil, err
		}
		return nil, result
	default:
		result := NewGetDashboardDefault(response.Code())
		if err := result.readResponse(response, consumer, o.formats); err != nil {
			return nil, err
		}
		if response.Code()/100 == 2 {
			return result, nil
		}
		return nil, result
	}
}

// NewGetDashboardOK creates a GetDashboardOK with default headers values
func NewGetDashboardOK() *GetDashboardOK {
	return &GetDashboardOK{}
}

/*
GetDashboardOK describes a response with status code 200, with default header values.

A successful response.
*/
type GetDashboardOK struct {
	Payload *service_model.V1Dashboard
}

// IsSuccess returns true when this get dashboard o k response has a 2xx status code
func (o *GetDashboardOK) IsSuccess() bool {
	return true
}

// IsRedirect returns true when this get dashboard o k response has a 3xx status code
func (o *GetDashboardOK) IsRedirect() bool {
	return false
}

// IsClientError returns true when this get dashboard o k response has a 4xx status code
func (o *GetDashboardOK) IsClientError() bool {
	return false
}

// IsServerError returns true when this get dashboard o k response has a 5xx status code
func (o *GetDashboardOK) IsServerError() bool {
	return false
}

// IsCode returns true when this get dashboard o k response a status code equal to that given
func (o *GetDashboardOK) IsCode(code int) bool {
	return code == 200
}

// Code gets the status code for the get dashboard o k response
func (o *GetDashboardOK) Code() int {
	return 200
}

func (o *GetDashboardOK) Error() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardOK  %+v", 200, o.Payload)
}

func (o *GetDashboardOK) String() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardOK  %+v", 200, o.Payload)
}

func (o *GetDashboardOK) GetPayload() *service_model.V1Dashboard {
	return o.Payload
}

func (o *GetDashboardOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {

	o.Payload = new(service_model.V1Dashboard)

	// response payload
	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
		return err
	}

	return nil
}

// NewGetDashboardNoContent creates a GetDashboardNoContent with default headers values
func NewGetDashboardNoContent() *GetDashboardNoContent {
	return &GetDashboardNoContent{}
}

/*
GetDashboardNoContent describes a response with status code 204, with default header values.

No content.
*/
type GetDashboardNoContent struct {
	Payload interface{}
}

// IsSuccess returns true when this get dashboard no content response has a 2xx status code
func (o *GetDashboardNoContent) IsSuccess() bool {
	return true
}

// IsRedirect returns true when this get dashboard no content response has a 3xx status code
func (o *GetDashboardNoContent) IsRedirect() bool {
	return false
}

// IsClientError returns true when this get dashboard no content response has a 4xx status code
func (o *GetDashboardNoContent) IsClientError() bool {
	return false
}

// IsServerError returns true when this get dashboard no content response has a 5xx status code
func (o *GetDashboardNoContent) IsServerError() bool {
	return false
}

// IsCode returns true when this get dashboard no content response a status code equal to that given
func (o *GetDashboardNoContent) IsCode(code int) bool {
	return code == 204
}

// Code gets the status code for the get dashboard no content response
func (o *GetDashboardNoContent) Code() int {
	return 204
}

func (o *GetDashboardNoContent) Error() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardNoContent  %+v", 204, o.Payload)
}

func (o *GetDashboardNoContent) String() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardNoContent  %+v", 204, o.Payload)
}

func (o *GetDashboardNoContent) GetPayload() interface{} {
	return o.Payload
}

func (o *GetDashboardNoContent) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {

	// response payload
	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
		return err
	}

	return nil
}

// NewGetDashboardForbidden creates a GetDashboardForbidden with default headers values
func NewGetDashboardForbidden() *GetDashboardForbidden {
	return &GetDashboardForbidden{}
}

/*
GetDashboardForbidden describes a response with status code 403, with default header values.

You don't have permission to access the resource.
*/
type GetDashboardForbidden struct {
	Payload interface{}
}

// IsSuccess returns true when this get dashboard forbidden response has a 2xx status code
func (o *GetDashboardForbidden) IsSuccess() bool {
	return false
}

// IsRedirect returns true when this get dashboard forbidden response has a 3xx status code
func (o *GetDashboardForbidden) IsRedirect() bool {
	return false
}

// IsClientError returns true when this get dashboard forbidden response has a 4xx status code
func (o *GetDashboardForbidden) IsClientError() bool {
	return true
}

// IsServerError returns true when this get dashboard forbidden response has a 5xx status code
func (o *GetDashboardForbidden) IsServerError() bool {
	return false
}

// IsCode returns true when this get dashboard forbidden response a status code equal to that given
func (o *GetDashboardForbidden) IsCode(code int) bool {
	return code == 403
}

// Code gets the status code for the get dashboard forbidden response
func (o *GetDashboardForbidden) Code() int {
	return 403
}

func (o *GetDashboardForbidden) Error() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardForbidden  %+v", 403, o.Payload)
}

func (o *GetDashboardForbidden) String() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardForbidden  %+v", 403, o.Payload)
}

func (o *GetDashboardForbidden) GetPayload() interface{} {
	return o.Payload
}

func (o *GetDashboardForbidden) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {

	// response payload
	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
		return err
	}

	return nil
}

// NewGetDashboardNotFound creates a GetDashboardNotFound with default headers values
func NewGetDashboardNotFound() *GetDashboardNotFound {
	return &GetDashboardNotFound{}
}

/*
GetDashboardNotFound describes a response with status code 404, with default header values.

Resource does not exist.
*/
type GetDashboardNotFound struct {
	Payload interface{}
}

// IsSuccess returns true when this get dashboard not found response has a 2xx status code
func (o *GetDashboardNotFound) IsSuccess() bool {
	return false
}

// IsRedirect returns true when this get dashboard not found response has a 3xx status code
func (o *GetDashboardNotFound) IsRedirect() bool {
	return false
}

// IsClientError returns true when this get dashboard not found response has a 4xx status code
func (o *GetDashboardNotFound) IsClientError() bool {
	return true
}

// IsServerError returns true when this get dashboard not found response has a 5xx status code
func (o *GetDashboardNotFound) IsServerError() bool {
	return false
}

// IsCode returns true when this get dashboard not found response a status code equal to that given
func (o *GetDashboardNotFound) IsCode(code int) bool {
	return code == 404
}

// Code gets the status code for the get dashboard not found response
func (o *GetDashboardNotFound) Code() int {
	return 404
}

func (o *GetDashboardNotFound) Error() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardNotFound  %+v", 404, o.Payload)
}

func (o *GetDashboardNotFound) String() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] getDashboardNotFound  %+v", 404, o.Payload)
}

func (o *GetDashboardNotFound) GetPayload() interface{} {
	return o.Payload
}

func (o *GetDashboardNotFound) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {

	// response payload
	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
		return err
	}

	return nil
}

// NewGetDashboardDefault creates a GetDashboardDefault with default headers values
func NewGetDashboardDefault(code int) *GetDashboardDefault {
	return &GetDashboardDefault{
		_statusCode: code,
	}
}

/*
GetDashboardDefault describes a response with status code -1, with default header values.

An unexpected error response.
*/
type GetDashboardDefault struct {
	_statusCode int

	Payload *service_model.RuntimeError
}

// IsSuccess returns true when this get dashboard default response has a 2xx status code
func (o *GetDashboardDefault) IsSuccess() bool {
	return o._statusCode/100 == 2
}

// IsRedirect returns true when this get dashboard default response has a 3xx status code
func (o *GetDashboardDefault) IsRedirect() bool {
	return o._statusCode/100 == 3
}

// IsClientError returns true when this get dashboard default response has a 4xx status code
func (o *GetDashboardDefault) IsClientError() bool {
	return o._statusCode/100 == 4
}

// IsServerError returns true when this get dashboard default response has a 5xx status code
func (o *GetDashboardDefault) IsServerError() bool {
	return o._statusCode/100 == 5
}

// IsCode returns true when this get dashboard default response a status code equal to that given
func (o *GetDashboardDefault) IsCode(code int) bool {
	return o._statusCode == code
}

// Code gets the status code for the get dashboard default response
func (o *GetDashboardDefault) Code() int {
	return o._statusCode
}

func (o *GetDashboardDefault) Error() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] GetDashboard default  %+v", o._statusCode, o.Payload)
}

func (o *GetDashboardDefault) String() string {
	return fmt.Sprintf("[GET /api/v1/orgs/{owner}/dashboards/{uuid}][%d] GetDashboard default  %+v", o._statusCode, o.Payload)
}

func (o *GetDashboardDefault) GetPayload() *service_model.RuntimeError {
	return o.Payload
}

func (o *GetDashboardDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {

	o.Payload = new(service_model.RuntimeError)

	// response payload
	if err := consumer.Consume(response.Body(), o.Payload); err != nil && err != io.EOF {
		return err
	}

	return nil
}

repository: polyaxon/sdks

repo_name: polyaxon/sdks

gha_language: Java


text:
package parser

import (
	"strings"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/hclparse"
)

func Load(src []byte, name string) (hcl.Body, hcl.Diagnostics) {
	parser := hclparse.NewParser()

	var file *hcl.File
	var diags hcl.Diagnostics

	if strings.HasSuffix(name, ".json") {
		file, diags = parser.ParseJSON(src, name)
	} else {
		file, diags = parser.ParseHCL(src, name)
	}

	if file == nil || file.Body == nil {
		return hcl.EmptyBody(), diags
	}
	return file.Body, diags
}

repository: Buphido/couper

repo_name: Buphido/couper


text:
require recipes-kernel/linux-libc-headers/linux-libc-headers.inc

EXTRA_OEMAKE = "-e MAKEFLAGS="
KORG_ARCHIVE_COMPRESSION = "xz"
COMPATIBLE_MACHINE = "(alt|gose|koelsch|lager|porter|silk)"
PV_append = "+git${SRCREV}"

FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"

RENESAS_BACKPORTS_URL="git://git.kernel.org/pub/scm/linux/kernel/git/horms/renesas-backport.git"
SRCREV = "165e12ce2d7839e755debbec78dfa43b54345275"
SRC_URI = "${RENESAS_BACKPORTS_URL};protocol=git;branch=bsp/v3.10.31-ltsi/rcar-gen2-1.9.7 \
          file://scripts-Makefile.headersinst-install-headers-from-sc.patch \
          file://0001-kernel-add-support-for-gcc-5.patch \
          file://0001-kernel-add-support-for-gcc-6.patch \
"
S = "${WORKDIR}/git"

gha_language: Python