github.com/prebid/prebid-server/v2@v2.18.0/endpoints/openrtb2/auction.go (about)

     1  package openrtb2
     2  
     3  import (
     4  	"compress/gzip"
     5  	"context"
     6  	"encoding/json"
     7  	"errors"
     8  	"fmt"
     9  	"io"
    10  	"net/http"
    11  	"net/url"
    12  	"regexp"
    13  	"strconv"
    14  	"strings"
    15  	"time"
    16  
    17  	"github.com/buger/jsonparser"
    18  	"github.com/gofrs/uuid"
    19  	"github.com/golang/glog"
    20  	"github.com/julienschmidt/httprouter"
    21  	gpplib "github.com/prebid/go-gpp"
    22  	"github.com/prebid/go-gpp/constants"
    23  	"github.com/prebid/openrtb/v20/adcom1"
    24  	"github.com/prebid/openrtb/v20/native1"
    25  	nativeRequests "github.com/prebid/openrtb/v20/native1/request"
    26  	"github.com/prebid/openrtb/v20/openrtb2"
    27  	"github.com/prebid/openrtb/v20/openrtb3"
    28  	"github.com/prebid/prebid-server/v2/bidadjustment"
    29  	"github.com/prebid/prebid-server/v2/hooks"
    30  	"github.com/prebid/prebid-server/v2/ortb"
    31  	"github.com/prebid/prebid-server/v2/privacy"
    32  	"github.com/prebid/prebid-server/v2/privacysandbox"
    33  	"golang.org/x/net/publicsuffix"
    34  	jsonpatch "gopkg.in/evanphx/json-patch.v4"
    35  
    36  	accountService "github.com/prebid/prebid-server/v2/account"
    37  	"github.com/prebid/prebid-server/v2/analytics"
    38  	"github.com/prebid/prebid-server/v2/config"
    39  	"github.com/prebid/prebid-server/v2/currency"
    40  	"github.com/prebid/prebid-server/v2/errortypes"
    41  	"github.com/prebid/prebid-server/v2/exchange"
    42  	"github.com/prebid/prebid-server/v2/gdpr"
    43  	"github.com/prebid/prebid-server/v2/hooks/hookexecution"
    44  	"github.com/prebid/prebid-server/v2/metrics"
    45  	"github.com/prebid/prebid-server/v2/openrtb_ext"
    46  	"github.com/prebid/prebid-server/v2/prebid_cache_client"
    47  	"github.com/prebid/prebid-server/v2/privacy/ccpa"
    48  	"github.com/prebid/prebid-server/v2/privacy/lmt"
    49  	"github.com/prebid/prebid-server/v2/schain"
    50  	"github.com/prebid/prebid-server/v2/stored_requests"
    51  	"github.com/prebid/prebid-server/v2/stored_requests/backends/empty_fetcher"
    52  	"github.com/prebid/prebid-server/v2/stored_responses"
    53  	"github.com/prebid/prebid-server/v2/usersync"
    54  	"github.com/prebid/prebid-server/v2/util/httputil"
    55  	"github.com/prebid/prebid-server/v2/util/iputil"
    56  	"github.com/prebid/prebid-server/v2/util/jsonutil"
    57  	"github.com/prebid/prebid-server/v2/util/uuidutil"
    58  	"github.com/prebid/prebid-server/v2/version"
    59  )
    60  
    61  const ampChannel = "amp"
    62  const appChannel = "app"
    63  const secCookieDeprecation = "Sec-Cookie-Deprecation"
    64  const secBrowsingTopics = "Sec-Browsing-Topics"
    65  const observeBrowsingTopics = "Observe-Browsing-Topics"
    66  const observeBrowsingTopicsValue = "?1"
    67  
    68  var (
    69  	dntKey      string = http.CanonicalHeaderKey("DNT")
    70  	dntDisabled int8   = 0
    71  	dntEnabled  int8   = 1
    72  	notAmp      int8   = 0
    73  )
    74  
    75  var accountIdSearchPath = [...]struct {
    76  	isApp  bool
    77  	isDOOH bool
    78  	key    []string
    79  }{
    80  	{true, false, []string{"app", "publisher", "ext", openrtb_ext.PrebidExtKey, "parentAccount"}},
    81  	{true, false, []string{"app", "publisher", "id"}},
    82  	{false, false, []string{"site", "publisher", "ext", openrtb_ext.PrebidExtKey, "parentAccount"}},
    83  	{false, false, []string{"site", "publisher", "id"}},
    84  	{false, true, []string{"dooh", "publisher", "ext", openrtb_ext.PrebidExtKey, "parentAccount"}},
    85  	{false, true, []string{"dooh", "publisher", "id"}},
    86  }
    87  
    88  func NewEndpoint(
    89  	uuidGenerator uuidutil.UUIDGenerator,
    90  	ex exchange.Exchange,
    91  	validator openrtb_ext.BidderParamValidator,
    92  	requestsById stored_requests.Fetcher,
    93  	accounts stored_requests.AccountFetcher,
    94  	cfg *config.Configuration,
    95  	metricsEngine metrics.MetricsEngine,
    96  	analyticsRunner analytics.Runner,
    97  	disabledBidders map[string]string,
    98  	defReqJSON []byte,
    99  	bidderMap map[string]openrtb_ext.BidderName,
   100  	storedRespFetcher stored_requests.Fetcher,
   101  	hookExecutionPlanBuilder hooks.ExecutionPlanBuilder,
   102  	tmaxAdjustments *exchange.TmaxAdjustmentsPreprocessed,
   103  ) (httprouter.Handle, error) {
   104  	if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || metricsEngine == nil {
   105  		return nil, errors.New("NewEndpoint requires non-nil arguments.")
   106  	}
   107  
   108  	defRequest := len(defReqJSON) > 0
   109  
   110  	ipValidator := iputil.PublicNetworkIPValidator{
   111  		IPv4PrivateNetworks: cfg.RequestValidation.IPv4PrivateNetworksParsed,
   112  		IPv6PrivateNetworks: cfg.RequestValidation.IPv6PrivateNetworksParsed,
   113  	}
   114  
   115  	return httprouter.Handle((&endpointDeps{
   116  		uuidGenerator,
   117  		ex,
   118  		validator,
   119  		requestsById,
   120  		empty_fetcher.EmptyFetcher{},
   121  		accounts,
   122  		cfg,
   123  		metricsEngine,
   124  		analyticsRunner,
   125  		disabledBidders,
   126  		defRequest,
   127  		defReqJSON,
   128  		bidderMap,
   129  		nil,
   130  		nil,
   131  		ipValidator,
   132  		storedRespFetcher,
   133  		hookExecutionPlanBuilder,
   134  		tmaxAdjustments,
   135  		openrtb_ext.NormalizeBidderName}).Auction), nil
   136  }
   137  
   138  type normalizeBidderName func(name string) (openrtb_ext.BidderName, bool)
   139  
   140  type endpointDeps struct {
   141  	uuidGenerator             uuidutil.UUIDGenerator
   142  	ex                        exchange.Exchange
   143  	paramsValidator           openrtb_ext.BidderParamValidator
   144  	storedReqFetcher          stored_requests.Fetcher
   145  	videoFetcher              stored_requests.Fetcher
   146  	accounts                  stored_requests.AccountFetcher
   147  	cfg                       *config.Configuration
   148  	metricsEngine             metrics.MetricsEngine
   149  	analytics                 analytics.Runner
   150  	disabledBidders           map[string]string
   151  	defaultRequest            bool
   152  	defReqJSON                []byte
   153  	bidderMap                 map[string]openrtb_ext.BidderName
   154  	cache                     prebid_cache_client.Client
   155  	debugLogRegexp            *regexp.Regexp
   156  	privateNetworkIPValidator iputil.IPValidator
   157  	storedRespFetcher         stored_requests.Fetcher
   158  	hookExecutionPlanBuilder  hooks.ExecutionPlanBuilder
   159  	tmaxAdjustments           *exchange.TmaxAdjustmentsPreprocessed
   160  	normalizeBidderName       normalizeBidderName
   161  }
   162  
   163  func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
   164  	// Prebid Server interprets request.tmax to be the maximum amount of time that a caller is willing
   165  	// to wait for bids. However, tmax may be defined in the Stored Request data.
   166  	//
   167  	// If so, then the trip to the backend might use a significant amount of this time.
   168  	// We can respect timeouts more accurately if we note the *real* start time, and use it
   169  	// to compute the auction timeout.
   170  	start := time.Now()
   171  
   172  	hookExecutor := hookexecution.NewHookExecutor(deps.hookExecutionPlanBuilder, hookexecution.EndpointAuction, deps.metricsEngine)
   173  
   174  	ao := analytics.AuctionObject{
   175  		Status:    http.StatusOK,
   176  		Errors:    make([]error, 0),
   177  		StartTime: start,
   178  	}
   179  
   180  	labels := metrics.Labels{
   181  		Source:        metrics.DemandUnknown,
   182  		RType:         metrics.ReqTypeORTB2Web,
   183  		PubID:         metrics.PublisherUnknown,
   184  		CookieFlag:    metrics.CookieFlagUnknown,
   185  		RequestStatus: metrics.RequestStatusOK,
   186  	}
   187  
   188  	activityControl := privacy.ActivityControl{}
   189  	defer func() {
   190  		deps.metricsEngine.RecordRequest(labels)
   191  		deps.metricsEngine.RecordRequestTime(labels, time.Since(start))
   192  		deps.analytics.LogAuctionObject(&ao, activityControl)
   193  	}()
   194  
   195  	w.Header().Set("X-Prebid", version.BuildXPrebidHeader(version.Ver))
   196  	setBrowsingTopicsHeader(w, r)
   197  
   198  	req, impExtInfoMap, storedAuctionResponses, storedBidResponses, bidderImpReplaceImp, account, errL := deps.parseRequest(r, &labels, hookExecutor)
   199  	if errortypes.ContainsFatalError(errL) && writeError(errL, w, &labels) {
   200  		return
   201  	}
   202  
   203  	if rejectErr := hookexecution.FindFirstRejectOrNil(errL); rejectErr != nil {
   204  		ao.RequestWrapper = req
   205  		labels, ao = rejectAuctionRequest(*rejectErr, w, hookExecutor, req.BidRequest, account, labels, ao)
   206  		return
   207  	}
   208  
   209  	tcf2Config := gdpr.NewTCF2Config(deps.cfg.GDPR.TCF2, account.GDPR)
   210  
   211  	activityControl = privacy.NewActivityControl(&account.Privacy)
   212  
   213  	hookExecutor.SetActivityControl(activityControl)
   214  	hookExecutor.SetAccount(account)
   215  
   216  	ctx := context.Background()
   217  
   218  	timeout := deps.cfg.AuctionTimeouts.LimitAuctionTimeout(time.Duration(req.TMax) * time.Millisecond)
   219  	if timeout > 0 {
   220  		var cancel context.CancelFunc
   221  		ctx, cancel = context.WithDeadline(ctx, start.Add(timeout))
   222  		defer cancel()
   223  	}
   224  
   225  	// Read Usersyncs/Cookie
   226  	decoder := usersync.Base64Decoder{}
   227  	usersyncs := usersync.ReadCookie(r, decoder, &deps.cfg.HostCookie)
   228  	usersync.SyncHostCookie(r, usersyncs, &deps.cfg.HostCookie)
   229  
   230  	if req.Site != nil {
   231  		if usersyncs.HasAnyLiveSyncs() {
   232  			labels.CookieFlag = metrics.CookieFlagYes
   233  		} else {
   234  			labels.CookieFlag = metrics.CookieFlagNo
   235  		}
   236  	}
   237  
   238  	// Set Integration Information
   239  	err := deps.setIntegrationType(req, account)
   240  	if err != nil {
   241  		errL = append(errL, err)
   242  		writeError(errL, w, &labels)
   243  		return
   244  	}
   245  	secGPC := r.Header.Get("Sec-GPC")
   246  
   247  	warnings := errortypes.WarningOnly(errL)
   248  
   249  	auctionRequest := &exchange.AuctionRequest{
   250  		BidRequestWrapper:          req,
   251  		Account:                    *account,
   252  		UserSyncs:                  usersyncs,
   253  		RequestType:                labels.RType,
   254  		StartTime:                  start,
   255  		LegacyLabels:               labels,
   256  		Warnings:                   warnings,
   257  		GlobalPrivacyControlHeader: secGPC,
   258  		ImpExtInfoMap:              impExtInfoMap,
   259  		StoredAuctionResponses:     storedAuctionResponses,
   260  		StoredBidResponses:         storedBidResponses,
   261  		BidderImpReplaceImpID:      bidderImpReplaceImp,
   262  		PubID:                      labels.PubID,
   263  		HookExecutor:               hookExecutor,
   264  		TCF2Config:                 tcf2Config,
   265  		Activities:                 activityControl,
   266  		TmaxAdjustments:            deps.tmaxAdjustments,
   267  	}
   268  	auctionResponse, err := deps.ex.HoldAuction(ctx, auctionRequest, nil)
   269  	defer func() {
   270  		if !auctionRequest.BidderResponseStartTime.IsZero() {
   271  			deps.metricsEngine.RecordOverheadTime(metrics.MakeAuctionResponse, time.Since(auctionRequest.BidderResponseStartTime))
   272  		}
   273  	}()
   274  	ao.RequestWrapper = req
   275  	ao.Account = account
   276  	var response *openrtb2.BidResponse
   277  	if auctionResponse != nil {
   278  		response = auctionResponse.BidResponse
   279  	}
   280  	ao.Response = response
   281  	ao.SeatNonBid = auctionResponse.GetSeatNonBid()
   282  	rejectErr, isRejectErr := hookexecution.CastRejectErr(err)
   283  	if err != nil && !isRejectErr {
   284  		if errortypes.ReadCode(err) == errortypes.BadInputErrorCode {
   285  			writeError([]error{err}, w, &labels)
   286  			return
   287  		}
   288  		labels.RequestStatus = metrics.RequestStatusErr
   289  		w.WriteHeader(http.StatusInternalServerError)
   290  		fmt.Fprintf(w, "Critical error while running the auction: %v", err)
   291  		glog.Errorf("/openrtb2/auction Critical error: %v", err)
   292  		ao.Status = http.StatusInternalServerError
   293  		ao.Errors = append(ao.Errors, err)
   294  		return
   295  	} else if isRejectErr {
   296  		labels, ao = rejectAuctionRequest(*rejectErr, w, hookExecutor, req.BidRequest, account, labels, ao)
   297  		return
   298  	}
   299  
   300  	err = setSeatNonBidRaw(req, auctionResponse)
   301  	if err != nil {
   302  		glog.Errorf("Error setting seat non-bid: %v", err)
   303  	}
   304  	labels, ao = sendAuctionResponse(w, hookExecutor, response, req.BidRequest, account, labels, ao)
   305  }
   306  
   307  // setSeatNonBidRaw is transitional function for setting SeatNonBid inside bidResponse.Ext
   308  // Because,
   309  // 1. today exchange.HoldAuction prepares and marshals some piece of response.Ext which is then used by auction.go, amp_auction.go and video_auction.go
   310  // 2. As per discussion with Prebid Team we are planning to move away from - HoldAuction building openrtb2.BidResponse. instead respective auction modules will build this object
   311  // 3. So, we will need this method to do first,  unmarshalling of response.Ext
   312  func setSeatNonBidRaw(request *openrtb_ext.RequestWrapper, auctionResponse *exchange.AuctionResponse) error {
   313  	if auctionResponse == nil || auctionResponse.BidResponse == nil {
   314  		return nil
   315  	}
   316  	// unmarshalling is required here, until we are moving away from bidResponse.Ext, which is populated
   317  	// by HoldAuction
   318  	response := auctionResponse.BidResponse
   319  	respExt := &openrtb_ext.ExtBidResponse{}
   320  	if err := jsonutil.Unmarshal(response.Ext, &respExt); err != nil {
   321  		return err
   322  	}
   323  	if setSeatNonBid(respExt, request, auctionResponse) {
   324  		if respExtJson, err := jsonutil.Marshal(respExt); err == nil {
   325  			response.Ext = respExtJson
   326  			return nil
   327  		} else {
   328  			return err
   329  		}
   330  	}
   331  	return nil
   332  }
   333  
   334  func rejectAuctionRequest(
   335  	rejectErr hookexecution.RejectError,
   336  	w http.ResponseWriter,
   337  	hookExecutor hookexecution.HookStageExecutor,
   338  	request *openrtb2.BidRequest,
   339  	account *config.Account,
   340  	labels metrics.Labels,
   341  	ao analytics.AuctionObject,
   342  ) (metrics.Labels, analytics.AuctionObject) {
   343  	response := &openrtb2.BidResponse{NBR: openrtb3.NoBidReason(rejectErr.NBR).Ptr()}
   344  	if request != nil {
   345  		response.ID = request.ID
   346  	}
   347  
   348  	ao.Response = response
   349  	ao.Errors = append(ao.Errors, rejectErr)
   350  
   351  	return sendAuctionResponse(w, hookExecutor, response, request, account, labels, ao)
   352  }
   353  
   354  func sendAuctionResponse(
   355  	w http.ResponseWriter,
   356  	hookExecutor hookexecution.HookStageExecutor,
   357  	response *openrtb2.BidResponse,
   358  	request *openrtb2.BidRequest,
   359  	account *config.Account,
   360  	labels metrics.Labels,
   361  	ao analytics.AuctionObject,
   362  ) (metrics.Labels, analytics.AuctionObject) {
   363  	hookExecutor.ExecuteAuctionResponseStage(response)
   364  
   365  	if response != nil {
   366  		stageOutcomes := hookExecutor.GetOutcomes()
   367  		ao.HookExecutionOutcome = stageOutcomes
   368  
   369  		ext, warns, err := hookexecution.EnrichExtBidResponse(response.Ext, stageOutcomes, request, account)
   370  		if err != nil {
   371  			err = fmt.Errorf("Failed to enrich Bid Response with hook debug information: %s", err)
   372  			glog.Errorf(err.Error())
   373  			ao.Errors = append(ao.Errors, err)
   374  		} else {
   375  			response.Ext = ext
   376  		}
   377  
   378  		if len(warns) > 0 {
   379  			ao.Errors = append(ao.Errors, warns...)
   380  		}
   381  	}
   382  
   383  	// Fixes #231
   384  	enc := json.NewEncoder(w)
   385  	enc.SetEscapeHTML(false)
   386  
   387  	w.Header().Set("Content-Type", "application/json")
   388  
   389  	// If an error happens when encoding the response, there isn't much we can do.
   390  	// If we've sent _any_ bytes, then Go would have sent the 200 status code first.
   391  	// That status code can't be un-sent... so the best we can do is log the error.
   392  	if err := enc.Encode(response); err != nil {
   393  		labels.RequestStatus = metrics.RequestStatusNetworkErr
   394  		ao.Errors = append(ao.Errors, fmt.Errorf("/openrtb2/auction Failed to send response: %v", err))
   395  	}
   396  
   397  	return labels, ao
   398  }
   399  
   400  // setBrowsingTopicsHeader always set the Observe-Browsing-Topics header to a value of ?1 if the Sec-Browsing-Topics is present in request
   401  func setBrowsingTopicsHeader(w http.ResponseWriter, r *http.Request) {
   402  	if value := r.Header.Get(secBrowsingTopics); value != "" {
   403  		w.Header().Set(observeBrowsingTopics, observeBrowsingTopicsValue)
   404  	}
   405  }
   406  
   407  // parseRequest turns the HTTP request into an OpenRTB request. This is guaranteed to return:
   408  //
   409  //   - A context which times out appropriately, given the request.
   410  //   - A cancellation function which should be called if the auction finishes early.
   411  //
   412  // If the errors list is empty, then the returned request will be valid according to the OpenRTB 2.5 spec.
   413  // In case of "strong recommendations" in the spec, it tends to be restrictive. If a better workaround is
   414  // possible, it will return errors with messages that suggest improvements.
   415  //
   416  // If the errors list has at least one element, then no guarantees are made about the returned request.
   417  func (deps *endpointDeps) parseRequest(httpRequest *http.Request, labels *metrics.Labels, hookExecutor hookexecution.HookStageExecutor) (req *openrtb_ext.RequestWrapper, impExtInfoMap map[string]exchange.ImpExtInfo, storedAuctionResponses stored_responses.ImpsWithBidResponses, storedBidResponses stored_responses.ImpBidderStoredResp, bidderImpReplaceImpId stored_responses.BidderImpReplaceImpID, account *config.Account, errs []error) {
   418  	errs = nil
   419  	var err error
   420  	var errL []error
   421  	var r io.ReadCloser = httpRequest.Body
   422  	reqContentEncoding := httputil.ContentEncoding(httpRequest.Header.Get("Content-Encoding"))
   423  	if reqContentEncoding != "" {
   424  		if !deps.cfg.Compression.Request.IsSupported(reqContentEncoding) {
   425  			errs = []error{fmt.Errorf("Content-Encoding of type %s is not supported", reqContentEncoding)}
   426  			return
   427  		} else {
   428  			r, err = getCompressionEnabledReader(httpRequest.Body, reqContentEncoding)
   429  			if err != nil {
   430  				errs = []error{err}
   431  				return
   432  			}
   433  		}
   434  	}
   435  	defer r.Close()
   436  	limitedReqReader := &io.LimitedReader{
   437  		R: r,
   438  		N: deps.cfg.MaxRequestSize,
   439  	}
   440  
   441  	requestJson, err := io.ReadAll(limitedReqReader)
   442  	if err != nil {
   443  		errs = []error{err}
   444  		return
   445  	}
   446  
   447  	if limitedReqReader.N <= 0 {
   448  		// Limited Reader returns 0 if the request was exactly at the max size or over the limit.
   449  		// This is because it only reads up to N bytes. To check if the request was too large,
   450  		//  we need to look at the next byte of its underlying reader, limitedReader.R.
   451  		if _, err := limitedReqReader.R.Read(make([]byte, 1)); err != io.EOF {
   452  			// Discard the rest of the request body so that the connection can be reused.
   453  			io.Copy(io.Discard, httpRequest.Body)
   454  			errs = []error{fmt.Errorf("request size exceeded max size of %d bytes.", deps.cfg.MaxRequestSize)}
   455  			return
   456  		}
   457  	}
   458  
   459  	req = &openrtb_ext.RequestWrapper{}
   460  	req.BidRequest = &openrtb2.BidRequest{}
   461  
   462  	requestJson, rejectErr := hookExecutor.ExecuteEntrypointStage(httpRequest, requestJson)
   463  	if rejectErr != nil {
   464  		errs = []error{rejectErr}
   465  		if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
   466  			glog.Errorf("Failed to unmarshal BidRequest during entrypoint rejection: %s", err)
   467  		}
   468  		return
   469  	}
   470  
   471  	timeout := parseTimeout(requestJson, time.Duration(deps.cfg.StoredRequestsTimeout)*time.Millisecond)
   472  	ctx, cancel := context.WithTimeout(context.Background(), timeout)
   473  	defer cancel()
   474  
   475  	impInfo, errs := parseImpInfo(requestJson)
   476  	if len(errs) > 0 {
   477  		return nil, nil, nil, nil, nil, nil, errs
   478  	}
   479  
   480  	storedBidRequestId, hasStoredBidRequest, storedRequests, storedImps, errs := deps.getStoredRequests(ctx, requestJson, impInfo)
   481  	if len(errs) > 0 {
   482  		return
   483  	}
   484  
   485  	accountId, isAppReq, isDOOHReq, errs := getAccountIdFromRawRequest(hasStoredBidRequest, storedRequests[storedBidRequestId], requestJson)
   486  	// fill labels here in order to pass correct metrics in case of errors
   487  	if isAppReq {
   488  		labels.Source = metrics.DemandApp
   489  		labels.RType = metrics.ReqTypeORTB2App
   490  		labels.PubID = accountId
   491  	} else if isDOOHReq {
   492  		labels.Source = metrics.DemandDOOH
   493  		labels.RType = metrics.ReqTypeORTB2DOOH
   494  		labels.PubID = accountId
   495  	} else { // is Site request
   496  		labels.Source = metrics.DemandWeb
   497  		labels.PubID = accountId
   498  	}
   499  	if errs != nil {
   500  		return
   501  	}
   502  
   503  	// Look up account
   504  	account, errs = accountService.GetAccount(ctx, deps.cfg, deps.accounts, accountId, deps.metricsEngine)
   505  	if len(errs) > 0 {
   506  		return
   507  	}
   508  
   509  	hookExecutor.SetAccount(account)
   510  	requestJson, rejectErr = hookExecutor.ExecuteRawAuctionStage(requestJson)
   511  	if rejectErr != nil {
   512  		errs = []error{rejectErr}
   513  		if err = jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
   514  			glog.Errorf("Failed to unmarshal BidRequest during raw auction stage rejection: %s", err)
   515  		}
   516  		return
   517  	}
   518  
   519  	// retrieve storedRequests and storedImps once more in case stored data was changed by the raw auction hook
   520  	if hasPayloadUpdatesAt(hooks.StageRawAuctionRequest.String(), hookExecutor.GetOutcomes()) {
   521  		impInfo, errs = parseImpInfo(requestJson)
   522  		if len(errs) > 0 {
   523  			return nil, nil, nil, nil, nil, nil, errs
   524  		}
   525  		storedBidRequestId, hasStoredBidRequest, storedRequests, storedImps, errs = deps.getStoredRequests(ctx, requestJson, impInfo)
   526  		if len(errs) > 0 {
   527  			return
   528  		}
   529  	}
   530  
   531  	// Fetch the Stored Request data and merge it into the HTTP request.
   532  	if requestJson, impExtInfoMap, errs = deps.processStoredRequests(requestJson, impInfo, storedRequests, storedImps, storedBidRequestId, hasStoredBidRequest); len(errs) > 0 {
   533  		return
   534  	}
   535  
   536  	if err := jsonutil.UnmarshalValid(requestJson, req.BidRequest); err != nil {
   537  		errs = []error{err}
   538  		return
   539  	}
   540  
   541  	if err := mergeBidderParams(req); err != nil {
   542  		errs = []error{err}
   543  		return
   544  	}
   545  
   546  	// Populate any "missing" OpenRTB fields with info from other sources, (e.g. HTTP request headers).
   547  	if errsL := deps.setFieldsImplicitly(httpRequest, req, account); len(errsL) > 0 {
   548  		errs = append(errs, errsL...)
   549  	}
   550  
   551  	if err := ortb.SetDefaults(req); err != nil {
   552  		errs = []error{err}
   553  		return
   554  	}
   555  
   556  	if err := processInterstitials(req); err != nil {
   557  		errs = []error{err}
   558  		return
   559  	}
   560  
   561  	lmt.ModifyForIOS(req.BidRequest)
   562  
   563  	//Stored auction responses should be processed after stored requests due to possible impression modification
   564  	storedAuctionResponses, storedBidResponses, bidderImpReplaceImpId, errL = stored_responses.ProcessStoredResponses(ctx, req, deps.storedRespFetcher)
   565  	if len(errL) > 0 {
   566  		errs = append(errs, errL...)
   567  		return nil, nil, nil, nil, nil, nil, errs
   568  	}
   569  
   570  	hasStoredResponses := len(storedAuctionResponses) > 0
   571  	errL = deps.validateRequest(account, httpRequest, req, false, hasStoredResponses, storedBidResponses, hasStoredBidRequest)
   572  	if len(errL) > 0 {
   573  		errs = append(errs, errL...)
   574  	}
   575  
   576  	return
   577  }
   578  
   579  func getCompressionEnabledReader(body io.ReadCloser, contentEncoding httputil.ContentEncoding) (io.ReadCloser, error) {
   580  	switch contentEncoding {
   581  	case httputil.ContentEncodingGZIP:
   582  		return gzip.NewReader(body)
   583  	default:
   584  		return nil, fmt.Errorf("unsupported compression type '%s'", contentEncoding)
   585  	}
   586  }
   587  
   588  // hasPayloadUpdatesAt checks if there are any successful payload updates at given stage
   589  func hasPayloadUpdatesAt(stageName string, outcomes []hookexecution.StageOutcome) bool {
   590  	for _, outcome := range outcomes {
   591  		if stageName != outcome.Stage {
   592  			continue
   593  		}
   594  
   595  		for _, group := range outcome.Groups {
   596  			for _, invocationResult := range group.InvocationResults {
   597  				if invocationResult.Status == hookexecution.StatusSuccess &&
   598  					invocationResult.Action == hookexecution.ActionUpdate {
   599  					return true
   600  				}
   601  			}
   602  		}
   603  	}
   604  
   605  	return false
   606  }
   607  
   608  // parseTimeout returns parses tmax from the requestJson, or returns the default if it doesn't exist.
   609  //
   610  // requestJson should be the content of the POST body.
   611  //
   612  // If the request defines tmax explicitly, then this will return that duration in milliseconds.
   613  // If not, it will return the default timeout.
   614  func parseTimeout(requestJson []byte, defaultTimeout time.Duration) time.Duration {
   615  	if tmax, dataType, _, err := jsonparser.Get(requestJson, "tmax"); dataType != jsonparser.NotExist && err == nil {
   616  		if tmaxInt, err := strconv.Atoi(string(tmax)); err == nil && tmaxInt > 0 {
   617  			return time.Duration(tmaxInt) * time.Millisecond
   618  		}
   619  	}
   620  	return defaultTimeout
   621  }
   622  
   623  // mergeBidderParams merges bidder parameters in req.ext down to the imp[].ext level, with
   624  // priority given to imp[].ext in case of a conflict. No validation of bidder parameters or
   625  // of the ext json is performed. Unmarshal errors are not expected since the ext json was
   626  // validated during the bid request unmarshal.
   627  func mergeBidderParams(req *openrtb_ext.RequestWrapper) error {
   628  	reqExt, err := req.GetRequestExt()
   629  	if err != nil {
   630  		return nil
   631  	}
   632  
   633  	prebid := reqExt.GetPrebid()
   634  	if prebid == nil {
   635  		return nil
   636  	}
   637  
   638  	bidderParamsJson := prebid.BidderParams
   639  	if len(bidderParamsJson) == 0 {
   640  		return nil
   641  	}
   642  
   643  	bidderParams := map[string]map[string]json.RawMessage{}
   644  	if err := jsonutil.Unmarshal(bidderParamsJson, &bidderParams); err != nil {
   645  		return nil
   646  	}
   647  
   648  	for i, imp := range req.GetImp() {
   649  		impExt, err := imp.GetImpExt()
   650  		if err != nil {
   651  			continue
   652  		}
   653  
   654  		// merges bidder parameters passed at req.ext level with imp[].ext.BIDDER level
   655  		if err := mergeBidderParamsImpExt(impExt, bidderParams); err != nil {
   656  			return fmt.Errorf("error processing bidder parameters for imp[%d]: %s", i, err.Error())
   657  		}
   658  
   659  		// merges bidder parameters passed at req.ext level with imp[].ext.prebid.bidder.BIDDER level
   660  		if err := mergeBidderParamsImpExtPrebid(impExt, bidderParams); err != nil {
   661  			return fmt.Errorf("error processing bidder parameters for imp[%d]: %s", i, err.Error())
   662  		}
   663  	}
   664  
   665  	return nil
   666  }
   667  
   668  // mergeBidderParamsImpExt merges bidder parameters in req.ext down to the imp[].ext.BIDDER
   669  // level, giving priority to imp[].ext.BIDDER in case of a conflict. Unmarshal errors are not
   670  // expected since the ext json was validated during the bid request unmarshal.
   671  func mergeBidderParamsImpExt(impExt *openrtb_ext.ImpExt, reqExtParams map[string]map[string]json.RawMessage) error {
   672  	extMap := impExt.GetExt()
   673  	extMapModified := false
   674  
   675  	for bidder, params := range reqExtParams {
   676  		if !isPossibleBidder(bidder) {
   677  			continue
   678  		}
   679  
   680  		impExtBidder, impExtBidderExists := extMap[bidder]
   681  		if !impExtBidderExists || impExtBidder == nil {
   682  			continue
   683  		}
   684  
   685  		impExtBidderMap := map[string]json.RawMessage{}
   686  		if len(impExtBidder) > 0 {
   687  			if err := jsonutil.Unmarshal(impExtBidder, &impExtBidderMap); err != nil {
   688  				continue
   689  			}
   690  		}
   691  
   692  		modified := false
   693  		for key, value := range params {
   694  			if _, present := impExtBidderMap[key]; !present {
   695  				impExtBidderMap[key] = value
   696  				modified = true
   697  			}
   698  		}
   699  
   700  		if modified {
   701  			impExtBidderJson, err := jsonutil.Marshal(impExtBidderMap)
   702  			if err != nil {
   703  				return fmt.Errorf("error marshalling ext.BIDDER: %s", err.Error())
   704  			}
   705  			extMap[bidder] = impExtBidderJson
   706  			extMapModified = true
   707  		}
   708  	}
   709  
   710  	if extMapModified {
   711  		impExt.SetExt(extMap)
   712  	}
   713  
   714  	return nil
   715  }
   716  
   717  // mergeBidderParamsImpExtPrebid merges bidder parameters in req.ext down to the imp[].ext.prebid.bidder.BIDDER
   718  // level, giving priority to imp[].ext.prebid.bidder.BIDDER in case of a conflict.
   719  func mergeBidderParamsImpExtPrebid(impExt *openrtb_ext.ImpExt, reqExtParams map[string]map[string]json.RawMessage) error {
   720  	prebid := impExt.GetPrebid()
   721  	prebidModified := false
   722  
   723  	if prebid == nil || len(prebid.Bidder) == 0 {
   724  		return nil
   725  	}
   726  
   727  	for bidder, params := range reqExtParams {
   728  		impExtPrebidBidder, impExtPrebidBidderExists := prebid.Bidder[bidder]
   729  		if !impExtPrebidBidderExists || impExtPrebidBidder == nil {
   730  			continue
   731  		}
   732  
   733  		impExtPrebidBidderMap := map[string]json.RawMessage{}
   734  		if len(impExtPrebidBidder) > 0 {
   735  			if err := jsonutil.Unmarshal(impExtPrebidBidder, &impExtPrebidBidderMap); err != nil {
   736  				continue
   737  			}
   738  		}
   739  
   740  		modified := false
   741  		for key, value := range params {
   742  			if _, present := impExtPrebidBidderMap[key]; !present {
   743  				impExtPrebidBidderMap[key] = value
   744  				modified = true
   745  			}
   746  		}
   747  
   748  		if modified {
   749  			impExtPrebidBidderJson, err := jsonutil.Marshal(impExtPrebidBidderMap)
   750  			if err != nil {
   751  				return fmt.Errorf("error marshalling ext.prebid.bidder.BIDDER: %s", err.Error())
   752  			}
   753  			prebid.Bidder[bidder] = impExtPrebidBidderJson
   754  			prebidModified = true
   755  		}
   756  	}
   757  
   758  	if prebidModified {
   759  		impExt.SetPrebid(prebid)
   760  	}
   761  
   762  	return nil
   763  }
   764  
   765  func (deps *endpointDeps) validateRequest(account *config.Account, httpReq *http.Request, req *openrtb_ext.RequestWrapper, isAmp bool, hasStoredResponses bool, storedBidResp stored_responses.ImpBidderStoredResp, hasStoredBidRequest bool) []error {
   766  	errL := []error{}
   767  	if req.ID == "" {
   768  		return []error{errors.New("request missing required field: \"id\"")}
   769  	}
   770  
   771  	if req.TMax < 0 {
   772  		return []error{fmt.Errorf("request.tmax must be nonnegative. Got %d", req.TMax)}
   773  	}
   774  
   775  	if req.LenImp() < 1 {
   776  		return []error{errors.New("request.imp must contain at least one element.")}
   777  	}
   778  
   779  	if len(req.Cur) > 1 {
   780  		req.Cur = req.Cur[0:1]
   781  		errL = append(errL, &errortypes.Warning{Message: fmt.Sprintf("A prebid request can only process one currency. Taking the first currency in the list, %s, as the active currency", req.Cur[0])})
   782  	}
   783  
   784  	// If automatically filling source TID is enabled then validate that
   785  	// source.TID exists and If it doesn't, fill it with a randomly generated UUID
   786  	if deps.cfg.AutoGenSourceTID {
   787  		if err := validateAndFillSourceTID(req, deps.cfg.GenerateRequestID, hasStoredBidRequest, isAmp); err != nil {
   788  			return []error{err}
   789  		}
   790  	}
   791  
   792  	var requestAliases map[string]string
   793  	reqExt, err := req.GetRequestExt()
   794  	if err != nil {
   795  		return []error{fmt.Errorf("request.ext is invalid: %v", err)}
   796  	}
   797  
   798  	reqPrebid := reqExt.GetPrebid()
   799  	if err := deps.parseBidExt(req); err != nil {
   800  		return []error{err}
   801  	}
   802  
   803  	if reqPrebid != nil {
   804  		requestAliases = reqPrebid.Aliases
   805  
   806  		if err := deps.validateAliases(requestAliases); err != nil {
   807  			return []error{err}
   808  		}
   809  
   810  		if err := deps.validateAliasesGVLIDs(reqPrebid.AliasGVLIDs, requestAliases); err != nil {
   811  			return []error{err}
   812  		}
   813  
   814  		if err := deps.validateBidAdjustmentFactors(reqPrebid.BidAdjustmentFactors, requestAliases); err != nil {
   815  			return []error{err}
   816  		}
   817  
   818  		if err := validateSChains(reqPrebid.SChains); err != nil {
   819  			return []error{err}
   820  		}
   821  
   822  		if err := deps.validateEidPermissions(reqPrebid.Data, requestAliases); err != nil {
   823  			return []error{err}
   824  		}
   825  
   826  		if err := currency.ValidateCustomRates(reqPrebid.CurrencyConversions); err != nil {
   827  			return []error{err}
   828  		}
   829  	}
   830  
   831  	if err := mapSChains(req); err != nil {
   832  		return []error{err}
   833  	}
   834  
   835  	if err := validateOrFillChannel(req, isAmp); err != nil {
   836  		return []error{err}
   837  	}
   838  
   839  	if err := validateExactlyOneInventoryType(req); err != nil {
   840  		return []error{err}
   841  	}
   842  
   843  	if errs := validateRequestExt(req); len(errs) != 0 {
   844  		if errortypes.ContainsFatalError(errs) {
   845  			return append(errL, errs...)
   846  		}
   847  		errL = append(errL, errs...)
   848  	}
   849  
   850  	if err := deps.validateSite(req); err != nil {
   851  		return append(errL, err)
   852  	}
   853  
   854  	if err := deps.validateApp(req); err != nil {
   855  		return append(errL, err)
   856  	}
   857  
   858  	if err := deps.validateDOOH(req); err != nil {
   859  		return append(errL, err)
   860  	}
   861  	var gpp gpplib.GppContainer
   862  	if req.BidRequest.Regs != nil && len(req.BidRequest.Regs.GPP) > 0 {
   863  		var errs []error
   864  		gpp, errs = gpplib.Parse(req.BidRequest.Regs.GPP)
   865  		if len(errs) > 0 {
   866  			errL = append(errL, &errortypes.Warning{
   867  				Message:     fmt.Sprintf("GPP consent string is invalid and will be ignored. (%v)", errs[0]),
   868  				WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
   869  		}
   870  	}
   871  
   872  	if errs := deps.validateUser(req, requestAliases, gpp); errs != nil {
   873  		if len(errs) > 0 {
   874  			errL = append(errL, errs...)
   875  		}
   876  		if errortypes.ContainsFatalError(errs) {
   877  			return errL
   878  		}
   879  	}
   880  
   881  	if errs := validateRegs(req, gpp); errs != nil {
   882  		if len(errs) > 0 {
   883  			errL = append(errL, errs...)
   884  		}
   885  		if errortypes.ContainsFatalError(errs) {
   886  			return errL
   887  		}
   888  	}
   889  
   890  	if err := validateDevice(req.Device); err != nil {
   891  		return append(errL, err)
   892  	}
   893  
   894  	if err := validateOrFillCookieDeprecation(httpReq, req, account); err != nil {
   895  		errL = append(errL, err)
   896  	}
   897  
   898  	if ccpaPolicy, err := ccpa.ReadFromRequestWrapper(req, gpp); err != nil {
   899  		errL = append(errL, err)
   900  		if errortypes.ContainsFatalError([]error{err}) {
   901  			return errL
   902  		}
   903  	} else if _, err := ccpaPolicy.Parse(exchange.GetValidBidders(requestAliases)); err != nil {
   904  		if _, invalidConsent := err.(*errortypes.Warning); invalidConsent {
   905  			errL = append(errL, &errortypes.Warning{
   906  				Message:     fmt.Sprintf("CCPA consent is invalid and will be ignored. (%v)", err),
   907  				WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
   908  			regsExt, err := req.GetRegExt()
   909  			if err != nil {
   910  				return append(errL, err)
   911  			}
   912  			regsExt.SetUSPrivacy("")
   913  		} else {
   914  			return append(errL, err)
   915  		}
   916  	}
   917  
   918  	impIDs := make(map[string]int, req.LenImp())
   919  	for i, imp := range req.GetImp() {
   920  		// check for unique imp id
   921  		if firstIndex, ok := impIDs[imp.ID]; ok {
   922  			errL = append(errL, fmt.Errorf(`request.imp[%d].id and request.imp[%d].id are both "%s". Imp IDs must be unique.`, firstIndex, i, imp.ID))
   923  		}
   924  		impIDs[imp.ID] = i
   925  
   926  		errs := deps.validateImp(imp, requestAliases, i, hasStoredResponses, storedBidResp)
   927  		if len(errs) > 0 {
   928  			errL = append(errL, errs...)
   929  		}
   930  		if errortypes.ContainsFatalError(errs) {
   931  			return errL
   932  		}
   933  	}
   934  
   935  	return errL
   936  }
   937  
   938  // mapSChains maps an schain defined in an ORTB 2.4 location (req.ext.schain) to the ORTB 2.5 location
   939  // (req.source.ext.schain) if no ORTB 2.5 schain (req.source.ext.schain, req.ext.prebid.schains) exists.
   940  // An ORTB 2.4 schain is always deleted from the 2.4 location regardless of whether an ORTB 2.5 schain exists.
   941  func mapSChains(req *openrtb_ext.RequestWrapper) error {
   942  	reqExt, err := req.GetRequestExt()
   943  	if err != nil {
   944  		return fmt.Errorf("req.ext is invalid: %v", err)
   945  	}
   946  	sourceExt, err := req.GetSourceExt()
   947  	if err != nil {
   948  		return fmt.Errorf("source.ext is invalid: %v", err)
   949  	}
   950  
   951  	reqExtSChain := reqExt.GetSChain()
   952  	reqExt.SetSChain(nil)
   953  
   954  	if reqPrebid := reqExt.GetPrebid(); reqPrebid != nil && reqPrebid.SChains != nil {
   955  		return nil
   956  	} else if sourceExt.GetSChain() != nil {
   957  		return nil
   958  	} else if reqExtSChain != nil {
   959  		sourceExt.SetSChain(reqExtSChain)
   960  	}
   961  	return nil
   962  }
   963  
   964  func validateAndFillSourceTID(req *openrtb_ext.RequestWrapper, generateRequestID bool, hasStoredBidRequest bool, isAmp bool) error {
   965  	if req.Source == nil {
   966  		req.Source = &openrtb2.Source{}
   967  	}
   968  
   969  	if req.Source.TID == "" || req.Source.TID == "{{UUID}}" || (generateRequestID && (isAmp || hasStoredBidRequest)) {
   970  		rawUUID, err := uuid.NewV4()
   971  		if err != nil {
   972  			return errors.New("error creating a random UUID for source.tid")
   973  		}
   974  		req.Source.TID = rawUUID.String()
   975  	}
   976  
   977  	for _, impWrapper := range req.GetImp() {
   978  		ie, _ := impWrapper.GetImpExt()
   979  		if ie.GetTid() == "" || ie.GetTid() == "{{UUID}}" || (generateRequestID && (isAmp || hasStoredBidRequest)) {
   980  			rawUUID, err := uuid.NewV4()
   981  			if err != nil {
   982  				return errors.New("imp.ext.tid missing in the imp and error creating a random UID")
   983  			}
   984  			ie.SetTid(rawUUID.String())
   985  			impWrapper.RebuildImp()
   986  		}
   987  	}
   988  
   989  	return nil
   990  }
   991  
   992  func (deps *endpointDeps) validateBidAdjustmentFactors(adjustmentFactors map[string]float64, aliases map[string]string) error {
   993  	uniqueBidders := make(map[string]struct{})
   994  	for bidderToAdjust, adjustmentFactor := range adjustmentFactors {
   995  		if adjustmentFactor <= 0 {
   996  			return fmt.Errorf("request.ext.prebid.bidadjustmentfactors.%s must be a positive number. Got %f", bidderToAdjust, adjustmentFactor)
   997  		}
   998  
   999  		bidderName := bidderToAdjust
  1000  		normalizedCoreBidder, ok := openrtb_ext.NormalizeBidderName(bidderToAdjust)
  1001  		if ok {
  1002  			bidderName = normalizedCoreBidder.String()
  1003  		}
  1004  
  1005  		if _, exists := uniqueBidders[bidderName]; exists {
  1006  			return fmt.Errorf("cannot have multiple bidders that differ only in case style")
  1007  		} else {
  1008  			uniqueBidders[bidderName] = struct{}{}
  1009  		}
  1010  
  1011  		if _, isBidder := deps.bidderMap[bidderName]; !isBidder {
  1012  			if _, isAlias := aliases[bidderToAdjust]; !isAlias {
  1013  				return fmt.Errorf("request.ext.prebid.bidadjustmentfactors.%s is not a known bidder or alias", bidderToAdjust)
  1014  			}
  1015  		}
  1016  	}
  1017  	return nil
  1018  }
  1019  
  1020  func validateSChains(sChains []*openrtb_ext.ExtRequestPrebidSChain) error {
  1021  	_, err := schain.BidderToPrebidSChains(sChains)
  1022  	return err
  1023  }
  1024  
  1025  func (deps *endpointDeps) validateEidPermissions(prebid *openrtb_ext.ExtRequestPrebidData, requestAliases map[string]string) error {
  1026  	if prebid == nil {
  1027  		return nil
  1028  	}
  1029  
  1030  	uniqueSources := make(map[string]struct{}, len(prebid.EidPermissions))
  1031  	for i, eid := range prebid.EidPermissions {
  1032  		if len(eid.Source) == 0 {
  1033  			return fmt.Errorf(`request.ext.prebid.data.eidpermissions[%d] missing required field: "source"`, i)
  1034  		}
  1035  
  1036  		if _, exists := uniqueSources[eid.Source]; exists {
  1037  			return fmt.Errorf(`request.ext.prebid.data.eidpermissions[%d] duplicate entry with field: "source"`, i)
  1038  		}
  1039  		uniqueSources[eid.Source] = struct{}{}
  1040  
  1041  		if len(eid.Bidders) == 0 {
  1042  			return fmt.Errorf(`request.ext.prebid.data.eidpermissions[%d] missing or empty required field: "bidders"`, i)
  1043  		}
  1044  
  1045  		if err := deps.validateBidders(eid.Bidders, deps.bidderMap, requestAliases); err != nil {
  1046  			return fmt.Errorf(`request.ext.prebid.data.eidpermissions[%d] contains %v`, i, err)
  1047  		}
  1048  	}
  1049  
  1050  	return nil
  1051  }
  1052  
  1053  func (deps *endpointDeps) validateBidders(bidders []string, knownBidders map[string]openrtb_ext.BidderName, knownRequestAliases map[string]string) error {
  1054  	for _, bidder := range bidders {
  1055  		if bidder == "*" {
  1056  			if len(bidders) > 1 {
  1057  				return errors.New(`bidder wildcard "*" mixed with specific bidders`)
  1058  			}
  1059  		} else {
  1060  			bidderNormalized, _ := deps.normalizeBidderName(bidder)
  1061  			_, isCoreBidder := knownBidders[bidderNormalized.String()]
  1062  			_, isAlias := knownRequestAliases[bidder]
  1063  			if !isCoreBidder && !isAlias {
  1064  				return fmt.Errorf(`unrecognized bidder "%v"`, bidder)
  1065  			}
  1066  		}
  1067  	}
  1068  	return nil
  1069  }
  1070  
  1071  func (deps *endpointDeps) validateImp(imp *openrtb_ext.ImpWrapper, aliases map[string]string, index int, hasStoredResponses bool, storedBidResp stored_responses.ImpBidderStoredResp) []error {
  1072  	if imp.ID == "" {
  1073  		return []error{fmt.Errorf("request.imp[%d] missing required field: \"id\"", index)}
  1074  	}
  1075  
  1076  	if len(imp.Metric) != 0 {
  1077  		return []error{fmt.Errorf("request.imp[%d].metric is not yet supported by prebid-server. Support may be added in the future", index)}
  1078  	}
  1079  
  1080  	if imp.Banner == nil && imp.Video == nil && imp.Audio == nil && imp.Native == nil {
  1081  		return []error{fmt.Errorf("request.imp[%d] must contain at least one of \"banner\", \"video\", \"audio\", or \"native\"", index)}
  1082  	}
  1083  
  1084  	if err := validateBanner(imp.Banner, index, isInterstitial(imp)); err != nil {
  1085  		return []error{err}
  1086  	}
  1087  
  1088  	if err := validateVideo(imp.Video, index); err != nil {
  1089  		return []error{err}
  1090  	}
  1091  
  1092  	if err := validateAudio(imp.Audio, index); err != nil {
  1093  		return []error{err}
  1094  	}
  1095  
  1096  	if err := fillAndValidateNative(imp.Native, index); err != nil {
  1097  		return []error{err}
  1098  	}
  1099  
  1100  	if err := validatePmp(imp.PMP, index); err != nil {
  1101  		return []error{err}
  1102  	}
  1103  
  1104  	errL := deps.validateImpExt(imp, aliases, index, hasStoredResponses, storedBidResp)
  1105  	if len(errL) != 0 {
  1106  		return errL
  1107  	}
  1108  
  1109  	return nil
  1110  }
  1111  
  1112  func isInterstitial(imp *openrtb_ext.ImpWrapper) bool {
  1113  	return imp.Instl == 1
  1114  }
  1115  
  1116  func validateBanner(banner *openrtb2.Banner, impIndex int, isInterstitial bool) error {
  1117  	if banner == nil {
  1118  		return nil
  1119  	}
  1120  
  1121  	// The following fields were previously uints in the OpenRTB library we use, but have
  1122  	// since been changed to ints. We decided to maintain the non-negative check.
  1123  	if banner.W != nil && *banner.W < 0 {
  1124  		return fmt.Errorf("request.imp[%d].banner.w must be a positive number", impIndex)
  1125  	}
  1126  	if banner.H != nil && *banner.H < 0 {
  1127  		return fmt.Errorf("request.imp[%d].banner.h must be a positive number", impIndex)
  1128  	}
  1129  
  1130  	// The following fields are deprecated in the OpenRTB 2.5 spec but are still present
  1131  	// in the OpenRTB library we use. Enforce they are not specified.
  1132  	if banner.WMin != 0 {
  1133  		return fmt.Errorf("request.imp[%d].banner uses unsupported property: \"wmin\". Use the \"format\" array instead.", impIndex)
  1134  	}
  1135  	if banner.WMax != 0 {
  1136  		return fmt.Errorf("request.imp[%d].banner uses unsupported property: \"wmax\". Use the \"format\" array instead.", impIndex)
  1137  	}
  1138  	if banner.HMin != 0 {
  1139  		return fmt.Errorf("request.imp[%d].banner uses unsupported property: \"hmin\". Use the \"format\" array instead.", impIndex)
  1140  	}
  1141  	if banner.HMax != 0 {
  1142  		return fmt.Errorf("request.imp[%d].banner uses unsupported property: \"hmax\". Use the \"format\" array instead.", impIndex)
  1143  	}
  1144  
  1145  	hasRootSize := banner.H != nil && banner.W != nil && *banner.H > 0 && *banner.W > 0
  1146  	if !hasRootSize && len(banner.Format) == 0 && !isInterstitial {
  1147  		return fmt.Errorf("request.imp[%d].banner has no sizes. Define \"w\" and \"h\", or include \"format\" elements.", impIndex)
  1148  	}
  1149  
  1150  	for i, format := range banner.Format {
  1151  		if err := validateFormat(&format, impIndex, i); err != nil {
  1152  			return err
  1153  		}
  1154  	}
  1155  
  1156  	return nil
  1157  }
  1158  
  1159  func validateVideo(video *openrtb2.Video, impIndex int) error {
  1160  	if video == nil {
  1161  		return nil
  1162  	}
  1163  
  1164  	if len(video.MIMEs) < 1 {
  1165  		return fmt.Errorf("request.imp[%d].video.mimes must contain at least one supported MIME type", impIndex)
  1166  	}
  1167  
  1168  	// The following fields were previously uints in the OpenRTB library we use, but have
  1169  	// since been changed to ints. We decided to maintain the non-negative check.
  1170  	if video.W != nil && *video.W < 0 {
  1171  		return fmt.Errorf("request.imp[%d].video.w must be a positive number", impIndex)
  1172  	}
  1173  	if video.H != nil && *video.H < 0 {
  1174  		return fmt.Errorf("request.imp[%d].video.h must be a positive number", impIndex)
  1175  	}
  1176  	if video.MinBitRate < 0 {
  1177  		return fmt.Errorf("request.imp[%d].video.minbitrate must be a positive number", impIndex)
  1178  	}
  1179  	if video.MaxBitRate < 0 {
  1180  		return fmt.Errorf("request.imp[%d].video.maxbitrate must be a positive number", impIndex)
  1181  	}
  1182  
  1183  	return nil
  1184  }
  1185  
  1186  func validateAudio(audio *openrtb2.Audio, impIndex int) error {
  1187  	if audio == nil {
  1188  		return nil
  1189  	}
  1190  
  1191  	if len(audio.MIMEs) < 1 {
  1192  		return fmt.Errorf("request.imp[%d].audio.mimes must contain at least one supported MIME type", impIndex)
  1193  	}
  1194  
  1195  	// The following fields were previously uints in the OpenRTB library we use, but have
  1196  	// since been changed to ints. We decided to maintain the non-negative check.
  1197  	if audio.Sequence < 0 {
  1198  		return fmt.Errorf("request.imp[%d].audio.sequence must be a positive number", impIndex)
  1199  	}
  1200  	if audio.MaxSeq < 0 {
  1201  		return fmt.Errorf("request.imp[%d].audio.maxseq must be a positive number", impIndex)
  1202  	}
  1203  	if audio.MinBitrate < 0 {
  1204  		return fmt.Errorf("request.imp[%d].audio.minbitrate must be a positive number", impIndex)
  1205  	}
  1206  	if audio.MaxBitrate < 0 {
  1207  		return fmt.Errorf("request.imp[%d].audio.maxbitrate must be a positive number", impIndex)
  1208  	}
  1209  
  1210  	return nil
  1211  }
  1212  
  1213  // fillAndValidateNative validates the request, and assigns the Asset IDs as recommended by the Native v1.2 spec.
  1214  func fillAndValidateNative(n *openrtb2.Native, impIndex int) error {
  1215  	if n == nil {
  1216  		return nil
  1217  	}
  1218  
  1219  	if len(n.Request) == 0 {
  1220  		return fmt.Errorf("request.imp[%d].native missing required property \"request\"", impIndex)
  1221  	}
  1222  	var nativePayload nativeRequests.Request
  1223  	if err := jsonutil.UnmarshalValid(json.RawMessage(n.Request), &nativePayload); err != nil {
  1224  		return err
  1225  	}
  1226  
  1227  	if err := validateNativeContextTypes(nativePayload.Context, nativePayload.ContextSubType, impIndex); err != nil {
  1228  		return err
  1229  	}
  1230  	if err := validateNativePlacementType(nativePayload.PlcmtType, impIndex); err != nil {
  1231  		return err
  1232  	}
  1233  	if err := fillAndValidateNativeAssets(nativePayload.Assets, impIndex); err != nil {
  1234  		return err
  1235  	}
  1236  	if err := validateNativeEventTrackers(nativePayload.EventTrackers, impIndex); err != nil {
  1237  		return err
  1238  	}
  1239  
  1240  	serialized, err := jsonutil.Marshal(nativePayload)
  1241  	if err != nil {
  1242  		return err
  1243  	}
  1244  	n.Request = string(serialized)
  1245  	return nil
  1246  }
  1247  
  1248  func validateNativeContextTypes(cType native1.ContextType, cSubtype native1.ContextSubType, impIndex int) error {
  1249  	if cType == 0 {
  1250  		// Context is only recommended, so none is a valid type.
  1251  		return nil
  1252  	}
  1253  	if cType < native1.ContextTypeContent || (cType > native1.ContextTypeProduct && cType < openrtb_ext.NativeExchangeSpecificLowerBound) {
  1254  		return fmt.Errorf("request.imp[%d].native.request.context is invalid. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex)
  1255  	}
  1256  	if cSubtype < 0 {
  1257  		return fmt.Errorf("request.imp[%d].native.request.contextsubtype value can't be less than 0. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex)
  1258  	}
  1259  	if cSubtype == 0 {
  1260  		return nil
  1261  	}
  1262  	if cSubtype >= native1.ContextSubTypeGeneral && cSubtype <= native1.ContextSubTypeUserGenerated {
  1263  		if cType != native1.ContextTypeContent && cType < openrtb_ext.NativeExchangeSpecificLowerBound {
  1264  			return fmt.Errorf("request.imp[%d].native.request.context is %d, but contextsubtype is %d. This is an invalid combination. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex, cType, cSubtype)
  1265  		}
  1266  		return nil
  1267  	}
  1268  	if cSubtype >= native1.ContextSubTypeSocial && cSubtype <= native1.ContextSubTypeChat {
  1269  		if cType != native1.ContextTypeSocial && cType < openrtb_ext.NativeExchangeSpecificLowerBound {
  1270  			return fmt.Errorf("request.imp[%d].native.request.context is %d, but contextsubtype is %d. This is an invalid combination. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex, cType, cSubtype)
  1271  		}
  1272  		return nil
  1273  	}
  1274  	if cSubtype >= native1.ContextSubTypeSelling && cSubtype <= native1.ContextSubTypeProductReview {
  1275  		if cType != native1.ContextTypeProduct && cType < openrtb_ext.NativeExchangeSpecificLowerBound {
  1276  			return fmt.Errorf("request.imp[%d].native.request.context is %d, but contextsubtype is %d. This is an invalid combination. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex, cType, cSubtype)
  1277  		}
  1278  		return nil
  1279  	}
  1280  	if cSubtype >= openrtb_ext.NativeExchangeSpecificLowerBound {
  1281  		return nil
  1282  	}
  1283  
  1284  	return fmt.Errorf("request.imp[%d].native.request.contextsubtype is invalid. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=39", impIndex)
  1285  }
  1286  
  1287  func validateNativePlacementType(pt native1.PlacementType, impIndex int) error {
  1288  	if pt == 0 {
  1289  		// Placement Type is only recommended, not required.
  1290  		return nil
  1291  	}
  1292  	if pt < native1.PlacementTypeFeed || (pt > native1.PlacementTypeRecommendationWidget && pt < openrtb_ext.NativeExchangeSpecificLowerBound) {
  1293  		return fmt.Errorf("request.imp[%d].native.request.plcmttype is invalid. See https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=40", impIndex)
  1294  	}
  1295  	return nil
  1296  }
  1297  
  1298  func fillAndValidateNativeAssets(assets []nativeRequests.Asset, impIndex int) error {
  1299  	if len(assets) < 1 {
  1300  		return fmt.Errorf("request.imp[%d].native.request.assets must be an array containing at least one object", impIndex)
  1301  	}
  1302  
  1303  	assetIDs := make(map[int64]struct{}, len(assets))
  1304  
  1305  	// If none of the asset IDs are defined by the caller, then prebid server should assign its own unique IDs. But
  1306  	// if the caller did assign its own asset IDs, then prebid server will respect those IDs
  1307  	assignAssetIDs := true
  1308  	for i := 0; i < len(assets); i++ {
  1309  		assignAssetIDs = assignAssetIDs && (assets[i].ID == 0)
  1310  	}
  1311  
  1312  	for i := 0; i < len(assets); i++ {
  1313  		if err := validateNativeAsset(assets[i], impIndex, i); err != nil {
  1314  			return err
  1315  		}
  1316  
  1317  		if assignAssetIDs {
  1318  			assets[i].ID = int64(i)
  1319  			continue
  1320  		}
  1321  
  1322  		// Each asset should have a unique ID thats assigned by the caller
  1323  		if _, ok := assetIDs[assets[i].ID]; ok {
  1324  			return fmt.Errorf("request.imp[%d].native.request.assets[%d].id is already being used by another asset. Each asset ID must be unique.", impIndex, i)
  1325  		}
  1326  
  1327  		assetIDs[assets[i].ID] = struct{}{}
  1328  	}
  1329  
  1330  	return nil
  1331  }
  1332  
  1333  func validateNativeAsset(asset nativeRequests.Asset, impIndex int, assetIndex int) error {
  1334  	assetErr := "request.imp[%d].native.request.assets[%d] must define exactly one of {title, img, video, data}"
  1335  	foundType := false
  1336  
  1337  	if asset.Title != nil {
  1338  		foundType = true
  1339  		if err := validateNativeAssetTitle(asset.Title, impIndex, assetIndex); err != nil {
  1340  			return err
  1341  		}
  1342  	}
  1343  
  1344  	if asset.Img != nil {
  1345  		if foundType {
  1346  			return fmt.Errorf(assetErr, impIndex, assetIndex)
  1347  		}
  1348  		foundType = true
  1349  		if err := validateNativeAssetImage(asset.Img, impIndex, assetIndex); err != nil {
  1350  			return err
  1351  		}
  1352  	}
  1353  
  1354  	if asset.Video != nil {
  1355  		if foundType {
  1356  			return fmt.Errorf(assetErr, impIndex, assetIndex)
  1357  		}
  1358  		foundType = true
  1359  		if err := validateNativeAssetVideo(asset.Video, impIndex, assetIndex); err != nil {
  1360  			return err
  1361  		}
  1362  	}
  1363  
  1364  	if asset.Data != nil {
  1365  		if foundType {
  1366  			return fmt.Errorf(assetErr, impIndex, assetIndex)
  1367  		}
  1368  		foundType = true
  1369  		if err := validateNativeAssetData(asset.Data, impIndex, assetIndex); err != nil {
  1370  			return err
  1371  		}
  1372  	}
  1373  
  1374  	if !foundType {
  1375  		return fmt.Errorf(assetErr, impIndex, assetIndex)
  1376  	}
  1377  
  1378  	return nil
  1379  }
  1380  
  1381  func validateNativeEventTrackers(trackers []nativeRequests.EventTracker, impIndex int) error {
  1382  	for i := 0; i < len(trackers); i++ {
  1383  		if err := validateNativeEventTracker(trackers[i], impIndex, i); err != nil {
  1384  			return err
  1385  		}
  1386  	}
  1387  	return nil
  1388  }
  1389  
  1390  func validateNativeAssetTitle(title *nativeRequests.Title, impIndex int, assetIndex int) error {
  1391  	if title.Len < 1 {
  1392  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].title.len must be a positive number", impIndex, assetIndex)
  1393  	}
  1394  	return nil
  1395  }
  1396  
  1397  func validateNativeEventTracker(tracker nativeRequests.EventTracker, impIndex int, eventIndex int) error {
  1398  	if tracker.Event < native1.EventTypeImpression || (tracker.Event > native1.EventTypeViewableVideo50 && tracker.Event < openrtb_ext.NativeExchangeSpecificLowerBound) {
  1399  		return fmt.Errorf("request.imp[%d].native.request.eventtrackers[%d].event is invalid. See section 7.6: https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=43", impIndex, eventIndex)
  1400  	}
  1401  	if len(tracker.Methods) < 1 {
  1402  		return fmt.Errorf("request.imp[%d].native.request.eventtrackers[%d].method is required. See section 7.7: https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=43", impIndex, eventIndex)
  1403  	}
  1404  	for methodIndex, method := range tracker.Methods {
  1405  		if method < native1.EventTrackingMethodImage || (method > native1.EventTrackingMethodJS && method < openrtb_ext.NativeExchangeSpecificLowerBound) {
  1406  			return fmt.Errorf("request.imp[%d].native.request.eventtrackers[%d].methods[%d] is invalid. See section 7.7: https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=43", impIndex, eventIndex, methodIndex)
  1407  		}
  1408  	}
  1409  
  1410  	return nil
  1411  }
  1412  
  1413  func validateNativeAssetImage(img *nativeRequests.Image, impIndex int, assetIndex int) error {
  1414  	if img.W < 0 {
  1415  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].img.w must be a positive integer", impIndex, assetIndex)
  1416  	}
  1417  	if img.H < 0 {
  1418  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].img.h must be a positive integer", impIndex, assetIndex)
  1419  	}
  1420  	if img.WMin < 0 {
  1421  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].img.wmin must be a positive integer", impIndex, assetIndex)
  1422  	}
  1423  	if img.HMin < 0 {
  1424  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].img.hmin must be a positive integer", impIndex, assetIndex)
  1425  	}
  1426  	return nil
  1427  }
  1428  
  1429  func validateNativeAssetVideo(video *nativeRequests.Video, impIndex int, assetIndex int) error {
  1430  	if len(video.MIMEs) < 1 {
  1431  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].video.mimes must be an array with at least one MIME type", impIndex, assetIndex)
  1432  	}
  1433  	if video.MinDuration < 1 {
  1434  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].video.minduration must be a positive integer", impIndex, assetIndex)
  1435  	}
  1436  	if video.MaxDuration < 1 {
  1437  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].video.maxduration must be a positive integer", impIndex, assetIndex)
  1438  	}
  1439  	if err := validateNativeVideoProtocols(video.Protocols, impIndex, assetIndex); err != nil {
  1440  		return err
  1441  	}
  1442  
  1443  	return nil
  1444  }
  1445  
  1446  func validateNativeAssetData(data *nativeRequests.Data, impIndex int, assetIndex int) error {
  1447  	if data.Type < native1.DataAssetTypeSponsored || (data.Type > native1.DataAssetTypeCTAText && data.Type < 500) {
  1448  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].data.type is invalid. See section 7.4: https://iabtechlab.com/wp-content/uploads/2016/07/OpenRTB-Native-Ads-Specification-Final-1.2.pdf#page=40", impIndex, assetIndex)
  1449  	}
  1450  
  1451  	return nil
  1452  }
  1453  
  1454  func validateNativeVideoProtocols(protocols []adcom1.MediaCreativeSubtype, impIndex int, assetIndex int) error {
  1455  	if len(protocols) < 1 {
  1456  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].video.protocols must be an array with at least one element", impIndex, assetIndex)
  1457  	}
  1458  	for i := 0; i < len(protocols); i++ {
  1459  		if err := validateNativeVideoProtocol(protocols[i], impIndex, assetIndex, i); err != nil {
  1460  			return err
  1461  		}
  1462  	}
  1463  	return nil
  1464  }
  1465  
  1466  func validateNativeVideoProtocol(protocol adcom1.MediaCreativeSubtype, impIndex int, assetIndex int, protocolIndex int) error {
  1467  	if protocol < adcom1.CreativeVAST10 || protocol > adcom1.CreativeDAAST10Wrapper {
  1468  		return fmt.Errorf("request.imp[%d].native.request.assets[%d].video.protocols[%d] is invalid. See Section 5.8: https://www.iab.com/wp-content/uploads/2016/03/OpenRTB-API-Specification-Version-2-5-FINAL.pdf#page=52", impIndex, assetIndex, protocolIndex)
  1469  	}
  1470  	return nil
  1471  }
  1472  
  1473  func validateFormat(format *openrtb2.Format, impIndex, formatIndex int) error {
  1474  	usesHW := format.W != 0 || format.H != 0
  1475  	usesRatios := format.WMin != 0 || format.WRatio != 0 || format.HRatio != 0
  1476  
  1477  	// The following fields were previously uints in the OpenRTB library we use, but have
  1478  	// since been changed to ints. We decided to maintain the non-negative check.
  1479  	if format.W < 0 {
  1480  		return fmt.Errorf("request.imp[%d].banner.format[%d].w must be a positive number", impIndex, formatIndex)
  1481  	}
  1482  	if format.H < 0 {
  1483  		return fmt.Errorf("request.imp[%d].banner.format[%d].h must be a positive number", impIndex, formatIndex)
  1484  	}
  1485  	if format.WRatio < 0 {
  1486  		return fmt.Errorf("request.imp[%d].banner.format[%d].wratio must be a positive number", impIndex, formatIndex)
  1487  	}
  1488  	if format.HRatio < 0 {
  1489  		return fmt.Errorf("request.imp[%d].banner.format[%d].hratio must be a positive number", impIndex, formatIndex)
  1490  	}
  1491  	if format.WMin < 0 {
  1492  		return fmt.Errorf("request.imp[%d].banner.format[%d].wmin must be a positive number", impIndex, formatIndex)
  1493  	}
  1494  
  1495  	if usesHW && usesRatios {
  1496  		return fmt.Errorf("Request imp[%d].banner.format[%d] should define *either* {w, h} *or* {wmin, wratio, hratio}, but not both. If both are valid, send two \"format\" objects in the request.", impIndex, formatIndex)
  1497  	}
  1498  	if !usesHW && !usesRatios {
  1499  		return fmt.Errorf("Request imp[%d].banner.format[%d] should define *either* {w, h} (for static size requirements) *or* {wmin, wratio, hratio} (for flexible sizes) to be non-zero.", impIndex, formatIndex)
  1500  	}
  1501  	if usesHW && (format.W == 0 || format.H == 0) {
  1502  		return fmt.Errorf("Request imp[%d].banner.format[%d] must define non-zero \"h\" and \"w\" properties.", impIndex, formatIndex)
  1503  	}
  1504  	if usesRatios && (format.WMin == 0 || format.WRatio == 0 || format.HRatio == 0) {
  1505  		return fmt.Errorf("Request imp[%d].banner.format[%d] must define non-zero \"wmin\", \"wratio\", and \"hratio\" properties.", impIndex, formatIndex)
  1506  	}
  1507  	return nil
  1508  }
  1509  
  1510  func validatePmp(pmp *openrtb2.PMP, impIndex int) error {
  1511  	if pmp == nil {
  1512  		return nil
  1513  	}
  1514  
  1515  	for dealIndex, deal := range pmp.Deals {
  1516  		if deal.ID == "" {
  1517  			return fmt.Errorf("request.imp[%d].pmp.deals[%d] missing required field: \"id\"", impIndex, dealIndex)
  1518  		}
  1519  	}
  1520  	return nil
  1521  }
  1522  
  1523  func (deps *endpointDeps) validateImpExt(imp *openrtb_ext.ImpWrapper, aliases map[string]string, impIndex int, hasStoredResponses bool, storedBidResp stored_responses.ImpBidderStoredResp) []error {
  1524  	if len(imp.Ext) == 0 {
  1525  		return []error{fmt.Errorf("request.imp[%d].ext is required", impIndex)}
  1526  	}
  1527  
  1528  	impExt, err := imp.GetImpExt()
  1529  	if err != nil {
  1530  		return []error{err}
  1531  	}
  1532  
  1533  	prebid := impExt.GetOrCreatePrebid()
  1534  	prebidModified := false
  1535  
  1536  	if prebid.Bidder == nil {
  1537  		prebid.Bidder = make(map[string]json.RawMessage)
  1538  	}
  1539  
  1540  	ext := impExt.GetExt()
  1541  	extModified := false
  1542  
  1543  	// promote imp[].ext.BIDDER to newer imp[].ext.prebid.bidder.BIDDER location, with the later taking precedence
  1544  	for k, v := range ext {
  1545  		if isPossibleBidder(k) {
  1546  			if _, exists := prebid.Bidder[k]; !exists {
  1547  				prebid.Bidder[k] = v
  1548  				prebidModified = true
  1549  			}
  1550  			delete(ext, k)
  1551  			extModified = true
  1552  		}
  1553  	}
  1554  
  1555  	if hasStoredResponses && prebid.StoredAuctionResponse == nil {
  1556  		return []error{fmt.Errorf("request validation failed. The StoredAuctionResponse.ID field must be completely present with, or completely absent from, all impressions in request. No StoredAuctionResponse data found for request.imp[%d].ext.prebid \n", impIndex)}
  1557  	}
  1558  
  1559  	if err := deps.validateStoredBidRespAndImpExtBidders(prebid, storedBidResp, imp.ID); err != nil {
  1560  		return []error{err}
  1561  	}
  1562  
  1563  	errL := []error{}
  1564  
  1565  	for bidder, ext := range prebid.Bidder {
  1566  		coreBidder, _ := openrtb_ext.NormalizeBidderName(bidder)
  1567  		if tmp, isAlias := aliases[bidder]; isAlias {
  1568  			coreBidder = openrtb_ext.BidderName(tmp)
  1569  		}
  1570  
  1571  		if coreBidderNormalized, isValid := deps.bidderMap[coreBidder.String()]; isValid {
  1572  			if err := deps.paramsValidator.Validate(coreBidderNormalized, ext); err != nil {
  1573  				return []error{fmt.Errorf("request.imp[%d].ext.prebid.bidder.%s failed validation.\n%v", impIndex, bidder, err)}
  1574  			}
  1575  		} else {
  1576  			if msg, isDisabled := deps.disabledBidders[bidder]; isDisabled {
  1577  				errL = append(errL, &errortypes.BidderTemporarilyDisabled{Message: msg})
  1578  				delete(prebid.Bidder, bidder)
  1579  				prebidModified = true
  1580  			} else {
  1581  				return []error{fmt.Errorf("request.imp[%d].ext.prebid.bidder contains unknown bidder: %s. Did you forget an alias in request.ext.prebid.aliases?", impIndex, bidder)}
  1582  			}
  1583  		}
  1584  	}
  1585  
  1586  	if len(prebid.Bidder) == 0 {
  1587  		errL = append(errL, fmt.Errorf("request.imp[%d].ext.prebid.bidder must contain at least one bidder", impIndex))
  1588  		return errL
  1589  	}
  1590  
  1591  	if prebidModified {
  1592  		impExt.SetPrebid(prebid)
  1593  	}
  1594  	if extModified {
  1595  		impExt.SetExt(ext)
  1596  	}
  1597  
  1598  	return errL
  1599  }
  1600  
  1601  // isPossibleBidder determines if a bidder name is a potential real bidder.
  1602  func isPossibleBidder(bidder string) bool {
  1603  	switch openrtb_ext.BidderName(bidder) {
  1604  	case openrtb_ext.BidderReservedContext:
  1605  		return false
  1606  	case openrtb_ext.BidderReservedData:
  1607  		return false
  1608  	case openrtb_ext.BidderReservedGPID:
  1609  		return false
  1610  	case openrtb_ext.BidderReservedPrebid:
  1611  		return false
  1612  	case openrtb_ext.BidderReservedSKAdN:
  1613  		return false
  1614  	case openrtb_ext.BidderReservedTID:
  1615  		return false
  1616  	case openrtb_ext.BidderReservedAE:
  1617  		return false
  1618  	default:
  1619  		return true
  1620  	}
  1621  }
  1622  
  1623  func (deps *endpointDeps) parseBidExt(req *openrtb_ext.RequestWrapper) error {
  1624  	if _, err := req.GetRequestExt(); err != nil {
  1625  		return fmt.Errorf("request.ext is invalid: %v", err)
  1626  	}
  1627  	return nil
  1628  }
  1629  
  1630  func (deps *endpointDeps) validateAliases(aliases map[string]string) error {
  1631  	for alias, bidderName := range aliases {
  1632  		normalisedBidderName, _ := openrtb_ext.NormalizeBidderName(bidderName)
  1633  		coreBidderName := normalisedBidderName.String()
  1634  		if _, isCoreBidderDisabled := deps.disabledBidders[coreBidderName]; isCoreBidderDisabled {
  1635  			return fmt.Errorf("request.ext.prebid.aliases.%s refers to disabled bidder: %s", alias, bidderName)
  1636  		}
  1637  
  1638  		if _, isCoreBidder := deps.bidderMap[coreBidderName]; !isCoreBidder {
  1639  			return fmt.Errorf("request.ext.prebid.aliases.%s refers to unknown bidder: %s", alias, bidderName)
  1640  		}
  1641  
  1642  		if alias == coreBidderName {
  1643  			return fmt.Errorf("request.ext.prebid.aliases.%s defines a no-op alias. Choose a different alias, or remove this entry.", alias)
  1644  		}
  1645  		aliases[alias] = coreBidderName
  1646  	}
  1647  	return nil
  1648  }
  1649  
  1650  func (deps *endpointDeps) validateAliasesGVLIDs(aliasesGVLIDs map[string]uint16, aliases map[string]string) error {
  1651  	for alias, vendorId := range aliasesGVLIDs {
  1652  
  1653  		if _, aliasExist := aliases[alias]; !aliasExist {
  1654  			return fmt.Errorf("request.ext.prebid.aliasgvlids. vendorId %d refers to unknown bidder alias: %s", vendorId, alias)
  1655  		}
  1656  
  1657  		if vendorId < 1 {
  1658  			return fmt.Errorf("request.ext.prebid.aliasgvlids. Invalid vendorId %d for alias: %s. Choose a different vendorId, or remove this entry.", vendorId, alias)
  1659  		}
  1660  	}
  1661  	return nil
  1662  }
  1663  
  1664  func validateRequestExt(req *openrtb_ext.RequestWrapper) []error {
  1665  	reqExt, err := req.GetRequestExt()
  1666  	if err != nil {
  1667  		return []error{err}
  1668  	}
  1669  
  1670  	prebid := reqExt.GetPrebid()
  1671  	// exit early if there is no request.ext.prebid to validate
  1672  	if prebid == nil {
  1673  		return nil
  1674  	}
  1675  
  1676  	if prebid.Cache != nil {
  1677  		if prebid.Cache.Bids == nil && prebid.Cache.VastXML == nil {
  1678  			return []error{errors.New(`request.ext is invalid: request.ext.prebid.cache requires one of the "bids" or "vastxml" properties`)}
  1679  		}
  1680  	}
  1681  
  1682  	if err := validateTargeting(prebid.Targeting); err != nil {
  1683  		return []error{err}
  1684  	}
  1685  
  1686  	var errs []error
  1687  	if prebid.MultiBid != nil {
  1688  		validatedMultiBids, multBidErrs := openrtb_ext.ValidateAndBuildExtMultiBid(prebid)
  1689  
  1690  		for _, err := range multBidErrs {
  1691  			errs = append(errs, &errortypes.Warning{
  1692  				WarningCode: errortypes.MultiBidWarningCode,
  1693  				Message:     err.Error(),
  1694  			})
  1695  		}
  1696  
  1697  		// update the downstream multibid to avoid passing unvalidated ext to bidders, etc.
  1698  		prebid.MultiBid = validatedMultiBids
  1699  		reqExt.SetPrebid(prebid)
  1700  	}
  1701  
  1702  	if !bidadjustment.Validate(prebid.BidAdjustments) {
  1703  		prebid.BidAdjustments = nil
  1704  		reqExt.SetPrebid(prebid)
  1705  		errs = append(errs, &errortypes.Warning{
  1706  			WarningCode: errortypes.BidAdjustmentWarningCode,
  1707  			Message:     "bid adjustment from request was invalid",
  1708  		})
  1709  	}
  1710  
  1711  	return errs
  1712  }
  1713  
  1714  func validateTargeting(t *openrtb_ext.ExtRequestTargeting) error {
  1715  	if t != nil {
  1716  		if t.PriceGranularity != nil {
  1717  			if err := validatePriceGranularity(t.PriceGranularity); err != nil {
  1718  				return err
  1719  			}
  1720  		}
  1721  		if t.MediaTypePriceGranularity.Video != nil {
  1722  			if err := validatePriceGranularity(t.MediaTypePriceGranularity.Video); err != nil {
  1723  				return err
  1724  			}
  1725  		}
  1726  		if t.MediaTypePriceGranularity.Banner != nil {
  1727  			if err := validatePriceGranularity(t.MediaTypePriceGranularity.Banner); err != nil {
  1728  				return err
  1729  			}
  1730  		}
  1731  		if t.MediaTypePriceGranularity.Native != nil {
  1732  			if err := validatePriceGranularity(t.MediaTypePriceGranularity.Native); err != nil {
  1733  				return err
  1734  			}
  1735  		}
  1736  	}
  1737  	return nil
  1738  }
  1739  
  1740  func validatePriceGranularity(pg *openrtb_ext.PriceGranularity) error {
  1741  	if pg.Precision == nil {
  1742  		return errors.New("Price granularity error: precision is required")
  1743  	} else if *pg.Precision < 0 {
  1744  		return errors.New("Price granularity error: precision must be non-negative")
  1745  	} else if *pg.Precision > openrtb_ext.MaxDecimalFigures {
  1746  		return fmt.Errorf("Price granularity error: precision of more than %d significant figures is not supported", openrtb_ext.MaxDecimalFigures)
  1747  	}
  1748  
  1749  	var prevMax float64 = 0
  1750  	for _, gr := range pg.Ranges {
  1751  		if gr.Max <= prevMax {
  1752  			return errors.New(`Price granularity error: range list must be ordered with increasing "max"`)
  1753  		}
  1754  
  1755  		if gr.Increment <= 0.0 {
  1756  			return errors.New("Price granularity error: increment must be a nonzero positive number")
  1757  		}
  1758  		prevMax = gr.Max
  1759  	}
  1760  	return nil
  1761  }
  1762  
  1763  func (deps *endpointDeps) validateSite(req *openrtb_ext.RequestWrapper) error {
  1764  	if req.Site == nil {
  1765  		return nil
  1766  	}
  1767  
  1768  	if req.Site.ID == "" && req.Site.Page == "" {
  1769  		return errors.New("request.site should include at least one of request.site.id or request.site.page.")
  1770  	}
  1771  	siteExt, err := req.GetSiteExt()
  1772  	if err != nil {
  1773  		return err
  1774  	}
  1775  	siteAmp := siteExt.GetAmp()
  1776  	if siteAmp != nil && (*siteAmp < 0 || *siteAmp > 1) {
  1777  		return errors.New(`request.site.ext.amp must be either 1, 0, or undefined`)
  1778  	}
  1779  
  1780  	return nil
  1781  }
  1782  
  1783  func (deps *endpointDeps) validateApp(req *openrtb_ext.RequestWrapper) error {
  1784  	if req.App == nil {
  1785  		return nil
  1786  	}
  1787  
  1788  	if req.App.ID != "" {
  1789  		if _, found := deps.cfg.BlacklistedAppMap[req.App.ID]; found {
  1790  			return &errortypes.BlacklistedApp{Message: fmt.Sprintf("Prebid-server does not process requests from App ID: %s", req.App.ID)}
  1791  		}
  1792  	}
  1793  
  1794  	_, err := req.GetAppExt()
  1795  	return err
  1796  }
  1797  
  1798  func (deps *endpointDeps) validateDOOH(req *openrtb_ext.RequestWrapper) error {
  1799  	if req.DOOH == nil {
  1800  		return nil
  1801  	}
  1802  
  1803  	if req.DOOH.ID == "" && len(req.DOOH.VenueType) == 0 {
  1804  		return errors.New("request.dooh should include at least one of request.dooh.id or request.dooh.venuetype.")
  1805  	}
  1806  
  1807  	return nil
  1808  }
  1809  
  1810  func (deps *endpointDeps) validateUser(req *openrtb_ext.RequestWrapper, aliases map[string]string, gpp gpplib.GppContainer) []error {
  1811  	var errL []error
  1812  
  1813  	if req == nil || req.BidRequest == nil || req.BidRequest.User == nil {
  1814  		return nil
  1815  	}
  1816  	// The following fields were previously uints in the OpenRTB library we use, but have
  1817  	// since been changed to ints. We decided to maintain the non-negative check.
  1818  	if req.User.Geo != nil && req.User.Geo.Accuracy < 0 {
  1819  		return append(errL, errors.New("request.user.geo.accuracy must be a positive number"))
  1820  	}
  1821  
  1822  	if req.User.Consent != "" {
  1823  		for _, section := range gpp.Sections {
  1824  			if section.GetID() == constants.SectionTCFEU2 && section.GetValue() != req.User.Consent {
  1825  				errL = append(errL, &errortypes.Warning{
  1826  					Message:     "user.consent GDPR string conflicts with GPP (regs.gpp) GDPR string, using regs.gpp",
  1827  					WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
  1828  			}
  1829  		}
  1830  	}
  1831  	userExt, err := req.GetUserExt()
  1832  	if err != nil {
  1833  		return append(errL, fmt.Errorf("request.user.ext object is not valid: %v", err))
  1834  	}
  1835  
  1836  	// Check if the buyeruids are valid
  1837  	prebid := userExt.GetPrebid()
  1838  	if prebid != nil {
  1839  		if len(prebid.BuyerUIDs) < 1 {
  1840  			return append(errL, errors.New(`request.user.ext.prebid requires a "buyeruids" property with at least one ID defined. If none exist, then request.user.ext.prebid should not be defined.`))
  1841  		}
  1842  		for bidderName := range prebid.BuyerUIDs {
  1843  			normalizedCoreBidder, _ := deps.normalizeBidderName(bidderName)
  1844  			coreBidder := normalizedCoreBidder.String()
  1845  			if _, ok := deps.bidderMap[coreBidder]; !ok {
  1846  				if _, ok := aliases[bidderName]; !ok {
  1847  					return append(errL, fmt.Errorf("request.user.ext.%s is neither a known bidder name nor an alias in request.ext.prebid.aliases", bidderName))
  1848  				}
  1849  			}
  1850  		}
  1851  	}
  1852  
  1853  	// Check Universal User ID
  1854  	eids := userExt.GetEid()
  1855  	if eids != nil {
  1856  		eidsValue := *eids
  1857  		for eidIndex, eid := range eidsValue {
  1858  			if eid.Source == "" {
  1859  				return append(errL, fmt.Errorf("request.user.ext.eids[%d] missing required field: \"source\"", eidIndex))
  1860  			}
  1861  
  1862  			if len(eid.UIDs) == 0 {
  1863  				return append(errL, fmt.Errorf("request.user.ext.eids[%d].uids must contain at least one element or be undefined", eidIndex))
  1864  			}
  1865  
  1866  			for uidIndex, uid := range eid.UIDs {
  1867  				if uid.ID == "" {
  1868  					return append(errL, fmt.Errorf("request.user.ext.eids[%d].uids[%d] missing required field: \"id\"", eidIndex, uidIndex))
  1869  				}
  1870  			}
  1871  		}
  1872  	}
  1873  
  1874  	return errL
  1875  }
  1876  
  1877  func validateRegs(req *openrtb_ext.RequestWrapper, gpp gpplib.GppContainer) []error {
  1878  	var errL []error
  1879  
  1880  	if req == nil || req.BidRequest == nil || req.BidRequest.Regs == nil {
  1881  		return nil
  1882  	}
  1883  
  1884  	if req.BidRequest.Regs.GDPR != nil && req.BidRequest.Regs.GPPSID != nil {
  1885  		gdpr := int8(0)
  1886  		for _, id := range req.BidRequest.Regs.GPPSID {
  1887  			if id == int8(constants.SectionTCFEU2) {
  1888  				gdpr = 1
  1889  				break
  1890  			}
  1891  		}
  1892  		if gdpr != *req.BidRequest.Regs.GDPR {
  1893  			errL = append(errL, &errortypes.Warning{
  1894  				Message:     "regs.gdpr signal conflicts with GPP (regs.gpp_sid) and will be ignored",
  1895  				WarningCode: errortypes.InvalidPrivacyConsentWarningCode})
  1896  		}
  1897  	}
  1898  	regsExt, err := req.GetRegExt()
  1899  	if err != nil {
  1900  		return append(errL, fmt.Errorf("request.regs.ext is invalid: %v", err))
  1901  	}
  1902  
  1903  	gdpr := regsExt.GetGDPR()
  1904  	if gdpr != nil && *gdpr != 0 && *gdpr != 1 {
  1905  		return append(errL, errors.New("request.regs.ext.gdpr must be either 0 or 1"))
  1906  	}
  1907  
  1908  	return errL
  1909  }
  1910  
  1911  func validateDevice(device *openrtb2.Device) error {
  1912  	if device == nil {
  1913  		return nil
  1914  	}
  1915  
  1916  	// The following fields were previously uints in the OpenRTB library we use, but have
  1917  	// since been changed to ints. We decided to maintain the non-negative check.
  1918  	if device.W < 0 {
  1919  		return errors.New("request.device.w must be a positive number")
  1920  	}
  1921  	if device.H < 0 {
  1922  		return errors.New("request.device.h must be a positive number")
  1923  	}
  1924  	if device.PPI < 0 {
  1925  		return errors.New("request.device.ppi must be a positive number")
  1926  	}
  1927  	if device.Geo != nil && device.Geo.Accuracy < 0 {
  1928  		return errors.New("request.device.geo.accuracy must be a positive number")
  1929  	}
  1930  
  1931  	return nil
  1932  }
  1933  
  1934  func validateOrFillCookieDeprecation(httpReq *http.Request, req *openrtb_ext.RequestWrapper, account *config.Account) error {
  1935  	if account == nil || !account.Privacy.PrivacySandbox.CookieDeprecation.Enabled {
  1936  		return nil
  1937  	}
  1938  
  1939  	deviceExt, err := req.GetDeviceExt()
  1940  	if err != nil {
  1941  		return err
  1942  	}
  1943  
  1944  	if deviceExt.GetCDep() != "" {
  1945  		return nil
  1946  	}
  1947  
  1948  	secCookieDeprecation := httpReq.Header.Get(secCookieDeprecation)
  1949  	if secCookieDeprecation == "" {
  1950  		return nil
  1951  	}
  1952  	if len(secCookieDeprecation) > 100 {
  1953  		return &errortypes.Warning{
  1954  			Message:     "request.device.ext.cdep must not exceed 100 characters",
  1955  			WarningCode: errortypes.SecCookieDeprecationLenWarningCode,
  1956  		}
  1957  	}
  1958  
  1959  	deviceExt.SetCDep(secCookieDeprecation)
  1960  	return nil
  1961  }
  1962  
  1963  func validateExactlyOneInventoryType(reqWrapper *openrtb_ext.RequestWrapper) error {
  1964  
  1965  	// Prep for mutual exclusion check
  1966  	invTypeNumMatches := 0
  1967  	if reqWrapper.Site != nil {
  1968  		invTypeNumMatches++
  1969  	}
  1970  	if reqWrapper.App != nil {
  1971  		invTypeNumMatches++
  1972  	}
  1973  	if reqWrapper.DOOH != nil {
  1974  		invTypeNumMatches++
  1975  	}
  1976  
  1977  	if invTypeNumMatches == 0 {
  1978  		return errors.New("One of request.site or request.app or request.dooh must be defined")
  1979  	} else if invTypeNumMatches >= 2 {
  1980  		return errors.New("No more than one of request.site or request.app or request.dooh can be defined")
  1981  	} else {
  1982  		return nil
  1983  	}
  1984  
  1985  }
  1986  
  1987  func validateOrFillChannel(reqWrapper *openrtb_ext.RequestWrapper, isAmp bool) error {
  1988  	requestExt, err := reqWrapper.GetRequestExt()
  1989  	if err != nil {
  1990  		return err
  1991  	}
  1992  	requestPrebid := requestExt.GetPrebid()
  1993  
  1994  	if requestPrebid == nil || requestPrebid.Channel == nil {
  1995  		fillChannel(reqWrapper, isAmp)
  1996  	} else if requestPrebid.Channel.Name == "" {
  1997  		return errors.New("ext.prebid.channel.name can't be empty")
  1998  	}
  1999  	return nil
  2000  }
  2001  
  2002  func fillChannel(reqWrapper *openrtb_ext.RequestWrapper, isAmp bool) error {
  2003  	var channelName string
  2004  	requestExt, err := reqWrapper.GetRequestExt()
  2005  	if err != nil {
  2006  		return err
  2007  	}
  2008  	requestPrebid := requestExt.GetPrebid()
  2009  	if isAmp {
  2010  		channelName = ampChannel
  2011  	}
  2012  	if reqWrapper.App != nil {
  2013  		channelName = appChannel
  2014  	}
  2015  	if channelName != "" {
  2016  		if requestPrebid == nil {
  2017  			requestPrebid = &openrtb_ext.ExtRequestPrebid{}
  2018  		}
  2019  		requestPrebid.Channel = &openrtb_ext.ExtRequestPrebidChannel{Name: channelName}
  2020  		requestExt.SetPrebid(requestPrebid)
  2021  		reqWrapper.RebuildRequest()
  2022  	}
  2023  	return nil
  2024  
  2025  }
  2026  
  2027  func sanitizeRequest(r *openrtb_ext.RequestWrapper, ipValidator iputil.IPValidator) {
  2028  	if r.Device != nil {
  2029  		if ip, ver := iputil.ParseIP(r.Device.IP); ip == nil || ver != iputil.IPv4 || !ipValidator.IsValid(ip, ver) {
  2030  			r.Device.IP = ""
  2031  		}
  2032  
  2033  		if ip, ver := iputil.ParseIP(r.Device.IPv6); ip == nil || ver != iputil.IPv6 || !ipValidator.IsValid(ip, ver) {
  2034  			r.Device.IPv6 = ""
  2035  		}
  2036  	}
  2037  }
  2038  
  2039  // setFieldsImplicitly uses _implicit_ information from the httpReq to set values on bidReq.
  2040  // This function does not consume the request body, which was set explicitly, but infers certain
  2041  // OpenRTB properties from the headers and other implicit info.
  2042  //
  2043  // This function _should not_ override any fields which were defined explicitly by the caller in the request.
  2044  func (deps *endpointDeps) setFieldsImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper, account *config.Account) []error {
  2045  	sanitizeRequest(r, deps.privateNetworkIPValidator)
  2046  
  2047  	setDeviceImplicitly(httpReq, r, deps.privateNetworkIPValidator)
  2048  
  2049  	// Per the OpenRTB spec: A bid request must not contain more than one of Site|App|DOOH
  2050  	// Assume it's a site request if it's not declared as one of the other values
  2051  	if r.App == nil && r.DOOH == nil {
  2052  		setSiteImplicitly(httpReq, r)
  2053  	}
  2054  
  2055  	setAuctionTypeImplicitly(r)
  2056  
  2057  	errs := setSecBrowsingTopicsImplicitly(httpReq, r, account)
  2058  	return errs
  2059  }
  2060  
  2061  // setDeviceImplicitly uses implicit info from httpReq to populate bidReq.Device
  2062  func setDeviceImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper, ipValidtor iputil.IPValidator) {
  2063  	setIPImplicitly(httpReq, r, ipValidtor)
  2064  	setUAImplicitly(httpReq, r)
  2065  	setDoNotTrackImplicitly(httpReq, r)
  2066  }
  2067  
  2068  // setAuctionTypeImplicitly sets the auction type to 1 if it wasn't on the request,
  2069  // since header bidding is generally a first-price auction.
  2070  func setAuctionTypeImplicitly(r *openrtb_ext.RequestWrapper) {
  2071  	if r.AT == 0 {
  2072  		r.AT = 1
  2073  	}
  2074  }
  2075  
  2076  // setSecBrowsingTopicsImplicitly updates user.data with data from request header 'Sec-Browsing-Topics'
  2077  func setSecBrowsingTopicsImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper, account *config.Account) []error {
  2078  	secBrowsingTopics := httpReq.Header.Get(secBrowsingTopics)
  2079  	if secBrowsingTopics == "" {
  2080  		return nil
  2081  	}
  2082  
  2083  	// host must configure privacy sandbox
  2084  	if account == nil || account.Privacy.PrivacySandbox.TopicsDomain == "" {
  2085  		return nil
  2086  	}
  2087  
  2088  	topics, errs := privacysandbox.ParseTopicsFromHeader(secBrowsingTopics)
  2089  	if len(topics) == 0 {
  2090  		return errs
  2091  	}
  2092  
  2093  	if r.User == nil {
  2094  		r.User = &openrtb2.User{}
  2095  	}
  2096  
  2097  	r.User.Data = privacysandbox.UpdateUserDataWithTopics(r.User.Data, topics, account.Privacy.PrivacySandbox.TopicsDomain)
  2098  	return errs
  2099  }
  2100  
  2101  func setSiteImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper) {
  2102  	if r.Site == nil {
  2103  		r.Site = &openrtb2.Site{}
  2104  	}
  2105  
  2106  	referrerCandidate := httpReq.Referer()
  2107  	if referrerCandidate == "" && r.Site.Page != "" {
  2108  		referrerCandidate = r.Site.Page // If http referer is disabled and thus has empty value - use site.page instead
  2109  	}
  2110  
  2111  	if referrerCandidate != "" {
  2112  		setSitePageIfEmpty(r.Site, referrerCandidate)
  2113  		if parsedUrl, err := url.Parse(referrerCandidate); err == nil {
  2114  			setSiteDomainIfEmpty(r.Site, parsedUrl.Host)
  2115  			if publisherDomain, err := publicsuffix.EffectiveTLDPlusOne(parsedUrl.Host); err == nil {
  2116  				setSitePublisherDomainIfEmpty(r.Site, publisherDomain)
  2117  			}
  2118  		}
  2119  	}
  2120  
  2121  	if siteExt, err := r.GetSiteExt(); err == nil && siteExt.GetAmp() == nil {
  2122  		siteExt.SetAmp(&notAmp)
  2123  	}
  2124  
  2125  }
  2126  
  2127  func setSitePageIfEmpty(site *openrtb2.Site, sitePage string) {
  2128  	if site.Page == "" {
  2129  		site.Page = sitePage
  2130  	}
  2131  }
  2132  
  2133  func setSiteDomainIfEmpty(site *openrtb2.Site, siteDomain string) {
  2134  	if site.Domain == "" {
  2135  		site.Domain = siteDomain
  2136  	}
  2137  }
  2138  
  2139  func setSitePublisherDomainIfEmpty(site *openrtb2.Site, publisherDomain string) {
  2140  	if site.Publisher == nil {
  2141  		site.Publisher = &openrtb2.Publisher{}
  2142  	}
  2143  	if site.Publisher.Domain == "" {
  2144  		site.Publisher.Domain = publisherDomain
  2145  	}
  2146  }
  2147  
  2148  func getJsonSyntaxError(testJSON []byte) (bool, string) {
  2149  	type JsonNode struct {
  2150  		raw   *json.RawMessage
  2151  		doc   map[string]*JsonNode
  2152  		ary   []*JsonNode
  2153  		which int
  2154  	}
  2155  	type jNode map[string]*JsonNode
  2156  	docErrdoc := &jNode{}
  2157  	docErr := jsonutil.UnmarshalValid(testJSON, docErrdoc)
  2158  	if uerror, ok := docErr.(*json.SyntaxError); ok {
  2159  		err := fmt.Sprintf("%s at offset %v", uerror.Error(), uerror.Offset)
  2160  		return true, err
  2161  	}
  2162  	return false, ""
  2163  }
  2164  
  2165  func (deps *endpointDeps) getStoredRequests(ctx context.Context, requestJson []byte, impInfo []ImpExtPrebidData) (string, bool, map[string]json.RawMessage, map[string]json.RawMessage, []error) {
  2166  	// Parse the Stored Request IDs from the BidRequest and Imps.
  2167  	storedBidRequestId, hasStoredBidRequest, err := getStoredRequestId(requestJson)
  2168  	if err != nil {
  2169  		return "", false, nil, nil, []error{err}
  2170  	}
  2171  
  2172  	// Fetch the Stored Request data
  2173  	var storedReqIds []string
  2174  	if hasStoredBidRequest {
  2175  		storedReqIds = []string{storedBidRequestId}
  2176  	}
  2177  
  2178  	impStoredReqIds := make([]string, 0, len(impInfo))
  2179  	impStoredReqIdsUniqueTracker := make(map[string]struct{}, len(impInfo))
  2180  	for _, impData := range impInfo {
  2181  		if impData.ImpExtPrebid.StoredRequest != nil && len(impData.ImpExtPrebid.StoredRequest.ID) > 0 {
  2182  			storedImpId := impData.ImpExtPrebid.StoredRequest.ID
  2183  			if _, present := impStoredReqIdsUniqueTracker[storedImpId]; !present {
  2184  				impStoredReqIds = append(impStoredReqIds, storedImpId)
  2185  				impStoredReqIdsUniqueTracker[storedImpId] = struct{}{}
  2186  			}
  2187  		}
  2188  	}
  2189  
  2190  	storedRequests, storedImps, errs := deps.storedReqFetcher.FetchRequests(ctx, storedReqIds, impStoredReqIds)
  2191  	if len(errs) != 0 {
  2192  		return "", false, nil, nil, errs
  2193  	}
  2194  
  2195  	return storedBidRequestId, hasStoredBidRequest, storedRequests, storedImps, errs
  2196  }
  2197  
  2198  func (deps *endpointDeps) processStoredRequests(requestJson []byte, impInfo []ImpExtPrebidData, storedRequests map[string]json.RawMessage, storedImps map[string]json.RawMessage, storedBidRequestId string, hasStoredBidRequest bool) ([]byte, map[string]exchange.ImpExtInfo, []error) {
  2199  	bidRequestID, err := getBidRequestID(storedRequests[storedBidRequestId])
  2200  	if err != nil {
  2201  		return nil, nil, []error{err}
  2202  	}
  2203  
  2204  	// Apply the Stored BidRequest, if it exists
  2205  	resolvedRequest := requestJson
  2206  
  2207  	if hasStoredBidRequest {
  2208  		isAppRequest, err := checkIfAppRequest(requestJson)
  2209  		if err != nil {
  2210  			return nil, nil, []error{err}
  2211  		}
  2212  		if (deps.cfg.GenerateRequestID && isAppRequest) || bidRequestID == "{{UUID}}" {
  2213  			uuidPatch, err := generateUuidForBidRequest(deps.uuidGenerator)
  2214  			if err != nil {
  2215  				return nil, nil, []error{err}
  2216  			}
  2217  			uuidPatch, err = jsonpatch.MergePatch(storedRequests[storedBidRequestId], uuidPatch)
  2218  			if err != nil {
  2219  				errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId)
  2220  				return nil, nil, errL
  2221  			}
  2222  			resolvedRequest, err = jsonpatch.MergePatch(requestJson, uuidPatch)
  2223  			if err != nil {
  2224  				errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId)
  2225  				return nil, nil, errL
  2226  			}
  2227  		} else {
  2228  			resolvedRequest, err = jsonpatch.MergePatch(storedRequests[storedBidRequestId], requestJson)
  2229  			if err != nil {
  2230  				errL := storedRequestErrorChecker(requestJson, storedRequests, storedBidRequestId)
  2231  				return nil, nil, errL
  2232  			}
  2233  		}
  2234  	}
  2235  
  2236  	// Apply default aliases, if they are provided
  2237  	if deps.defaultRequest {
  2238  		aliasedRequest, err := jsonpatch.MergePatch(deps.defReqJSON, resolvedRequest)
  2239  		if err != nil {
  2240  			hasErr, Err := getJsonSyntaxError(resolvedRequest)
  2241  			if hasErr {
  2242  				err = fmt.Errorf("Invalid JSON in Incoming Request: %s", Err)
  2243  			} else {
  2244  				hasErr, Err = getJsonSyntaxError(deps.defReqJSON)
  2245  				if hasErr {
  2246  					err = fmt.Errorf("Invalid JSON in Default Request Settings: %s", Err)
  2247  				}
  2248  			}
  2249  			return nil, nil, []error{err}
  2250  		}
  2251  		resolvedRequest = aliasedRequest
  2252  	}
  2253  
  2254  	// Apply any Stored Imps, if they exist. Since the JSON Merge Patch overrides arrays,
  2255  	// and Prebid Server defers to the HTTP Request to resolve conflicts, it's safe to
  2256  	// assume that the request.imp data did not change when applying the Stored BidRequest.
  2257  	impExtInfoMap := make(map[string]exchange.ImpExtInfo, len(impInfo))
  2258  	resolvedImps := make([]json.RawMessage, 0, len(impInfo))
  2259  	for i, impData := range impInfo {
  2260  		if impData.ImpExtPrebid.StoredRequest != nil && len(impData.ImpExtPrebid.StoredRequest.ID) > 0 {
  2261  			resolvedImp, err := jsonpatch.MergePatch(storedImps[impData.ImpExtPrebid.StoredRequest.ID], impData.Imp)
  2262  
  2263  			if err != nil {
  2264  				hasErr, errMessage := getJsonSyntaxError(impData.Imp)
  2265  				if hasErr {
  2266  					err = fmt.Errorf("Invalid JSON in Imp[%d] of Incoming Request: %s", i, errMessage)
  2267  				} else {
  2268  					hasErr, errMessage = getJsonSyntaxError(storedImps[impData.ImpExtPrebid.StoredRequest.ID])
  2269  					if hasErr {
  2270  						err = fmt.Errorf("imp.ext.prebid.storedrequest.id %s: Stored Imp has Invalid JSON: %s", impData.ImpExtPrebid.StoredRequest.ID, errMessage)
  2271  					}
  2272  				}
  2273  				return nil, nil, []error{err}
  2274  			}
  2275  			resolvedImps = append(resolvedImps, resolvedImp)
  2276  			impId, err := jsonparser.GetString(resolvedImp, "id")
  2277  			if err != nil {
  2278  				return nil, nil, []error{err}
  2279  			}
  2280  
  2281  			echoVideoAttributes := false
  2282  			if impData.ImpExtPrebid.Options != nil {
  2283  				echoVideoAttributes = impData.ImpExtPrebid.Options.EchoVideoAttrs
  2284  			}
  2285  
  2286  			// Extract Passthrough from Merged Imp
  2287  			passthrough, _, _, err := jsonparser.Get(resolvedImp, "ext", "prebid", "passthrough")
  2288  			if err != nil && err != jsonparser.KeyPathNotFoundError {
  2289  				return nil, nil, []error{err}
  2290  			}
  2291  			impExtInfoMap[impId] = exchange.ImpExtInfo{EchoVideoAttrs: echoVideoAttributes, StoredImp: storedImps[impData.ImpExtPrebid.StoredRequest.ID], Passthrough: passthrough}
  2292  
  2293  		} else {
  2294  			resolvedImps = append(resolvedImps, impData.Imp)
  2295  			impId, err := jsonparser.GetString(impData.Imp, "id")
  2296  			if err != nil {
  2297  				if err == jsonparser.KeyPathNotFoundError {
  2298  					err = fmt.Errorf("request.imp[%d] missing required field: \"id\"\n", i)
  2299  				}
  2300  				return nil, nil, []error{err}
  2301  			}
  2302  			impExtInfoMap[impId] = exchange.ImpExtInfo{Passthrough: impData.ImpExtPrebid.Passthrough}
  2303  		}
  2304  	}
  2305  	if len(resolvedImps) > 0 {
  2306  		newImpJson, err := jsonutil.Marshal(resolvedImps)
  2307  		if err != nil {
  2308  			return nil, nil, []error{err}
  2309  		}
  2310  		resolvedRequest, err = jsonparser.Set(resolvedRequest, newImpJson, "imp")
  2311  		if err != nil {
  2312  			return nil, nil, []error{err}
  2313  		}
  2314  	}
  2315  
  2316  	return resolvedRequest, impExtInfoMap, nil
  2317  }
  2318  
  2319  // parseImpInfo parses the request JSON and returns impression and unmarshalled imp.ext.prebid
  2320  func parseImpInfo(requestJson []byte) (impData []ImpExtPrebidData, errs []error) {
  2321  	if impArray, dataType, _, err := jsonparser.Get(requestJson, "imp"); err == nil && dataType == jsonparser.Array {
  2322  		_, _ = jsonparser.ArrayEach(impArray, func(imp []byte, _ jsonparser.ValueType, _ int, _ error) {
  2323  			impExtData, _, _, _ := jsonparser.Get(imp, "ext", "prebid")
  2324  			var impExtPrebid openrtb_ext.ExtImpPrebid
  2325  			if impExtData != nil {
  2326  				if err := jsonutil.Unmarshal(impExtData, &impExtPrebid); err != nil {
  2327  					errs = append(errs, err)
  2328  				}
  2329  			}
  2330  			newImpData := ImpExtPrebidData{imp, impExtPrebid}
  2331  			impData = append(impData, newImpData)
  2332  		})
  2333  	}
  2334  	return
  2335  }
  2336  
  2337  type ImpExtPrebidData struct {
  2338  	Imp          json.RawMessage
  2339  	ImpExtPrebid openrtb_ext.ExtImpPrebid
  2340  }
  2341  
  2342  // getStoredRequestId parses a Stored Request ID from some json, without doing a full (slow) unmarshal.
  2343  // It returns the ID, true/false whether a stored request key existed, and an error if anything went wrong
  2344  // (e.g. malformed json, id not a string, etc).
  2345  func getStoredRequestId(data []byte) (string, bool, error) {
  2346  	// These keys must be kept in sync with openrtb_ext.ExtStoredRequest
  2347  	storedRequestId, dataType, _, err := jsonparser.Get(data, "ext", openrtb_ext.PrebidExtKey, "storedrequest", "id")
  2348  
  2349  	if dataType == jsonparser.NotExist {
  2350  		return "", false, nil
  2351  	}
  2352  	if err != nil {
  2353  		return "", false, err
  2354  	}
  2355  	if dataType != jsonparser.String {
  2356  		return "", true, errors.New("ext.prebid.storedrequest.id must be a string")
  2357  	}
  2358  	return string(storedRequestId), true, nil
  2359  }
  2360  
  2361  func getBidRequestID(data json.RawMessage) (string, error) {
  2362  	bidRequestID, dataType, _, err := jsonparser.Get(data, "id")
  2363  	if dataType == jsonparser.NotExist {
  2364  		return "", nil
  2365  	}
  2366  	if err != nil {
  2367  		return "", err
  2368  	}
  2369  	return string(bidRequestID), nil
  2370  }
  2371  
  2372  // setIPImplicitly sets the IP address on bidReq, if it's not explicitly defined and we can figure it out.
  2373  func setIPImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper, ipValidator iputil.IPValidator) {
  2374  	if r.Device == nil || (r.Device.IP == "" && r.Device.IPv6 == "") {
  2375  		if ip, ver := httputil.FindIP(httpReq, ipValidator); ip != nil {
  2376  			switch ver {
  2377  			case iputil.IPv4:
  2378  				if r.Device == nil {
  2379  					r.Device = &openrtb2.Device{}
  2380  				}
  2381  				r.Device.IP = ip.String()
  2382  			case iputil.IPv6:
  2383  				if r.Device == nil {
  2384  					r.Device = &openrtb2.Device{}
  2385  				}
  2386  				r.Device.IPv6 = ip.String()
  2387  			}
  2388  		}
  2389  	}
  2390  }
  2391  
  2392  // setUAImplicitly sets the User Agent on bidReq, if it's not explicitly defined and it's defined on the request.
  2393  func setUAImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper) {
  2394  	if r.Device == nil || r.Device.UA == "" {
  2395  		if ua := httpReq.UserAgent(); ua != "" {
  2396  			if r.Device == nil {
  2397  				r.Device = &openrtb2.Device{}
  2398  			}
  2399  			r.Device.UA = ua
  2400  		}
  2401  	}
  2402  }
  2403  
  2404  func setDoNotTrackImplicitly(httpReq *http.Request, r *openrtb_ext.RequestWrapper) {
  2405  	if r.Device == nil || r.Device.DNT == nil {
  2406  		dnt := httpReq.Header.Get(dntKey)
  2407  		if dnt == "0" || dnt == "1" {
  2408  			if r.Device == nil {
  2409  				r.Device = &openrtb2.Device{}
  2410  			}
  2411  
  2412  			switch dnt {
  2413  			case "0":
  2414  				r.Device.DNT = &dntDisabled
  2415  			case "1":
  2416  				r.Device.DNT = &dntEnabled
  2417  			}
  2418  		}
  2419  	}
  2420  }
  2421  
  2422  // Write(return) errors to the client, if any. Returns true if errors were found.
  2423  func writeError(errs []error, w http.ResponseWriter, labels *metrics.Labels) bool {
  2424  	var rc bool = false
  2425  	if len(errs) > 0 {
  2426  		httpStatus := http.StatusBadRequest
  2427  		metricsStatus := metrics.RequestStatusBadInput
  2428  		for _, err := range errs {
  2429  			erVal := errortypes.ReadCode(err)
  2430  			if erVal == errortypes.BlacklistedAppErrorCode || erVal == errortypes.AccountDisabledErrorCode {
  2431  				httpStatus = http.StatusServiceUnavailable
  2432  				metricsStatus = metrics.RequestStatusBlacklisted
  2433  				break
  2434  			} else if erVal == errortypes.MalformedAcctErrorCode {
  2435  				httpStatus = http.StatusInternalServerError
  2436  				metricsStatus = metrics.RequestStatusAccountConfigErr
  2437  				break
  2438  			}
  2439  		}
  2440  		w.WriteHeader(httpStatus)
  2441  		labels.RequestStatus = metricsStatus
  2442  		for _, err := range errs {
  2443  			fmt.Fprintf(w, "Invalid request: %s\n", err.Error())
  2444  		}
  2445  		rc = true
  2446  	}
  2447  	return rc
  2448  }
  2449  
  2450  // Returns the account ID for the request
  2451  func getAccountID(pub *openrtb2.Publisher) string {
  2452  	if pub != nil {
  2453  		if pub.Ext != nil {
  2454  			var pubExt openrtb_ext.ExtPublisher
  2455  			err := jsonutil.Unmarshal(pub.Ext, &pubExt)
  2456  			if err == nil && pubExt.Prebid != nil && pubExt.Prebid.ParentAccount != nil && *pubExt.Prebid.ParentAccount != "" {
  2457  				return *pubExt.Prebid.ParentAccount
  2458  			}
  2459  		}
  2460  		if pub.ID != "" {
  2461  			return pub.ID
  2462  		}
  2463  	}
  2464  	return metrics.PublisherUnknown
  2465  }
  2466  
  2467  func getAccountIdFromRawRequest(hasStoredRequest bool, storedRequest json.RawMessage, originalRequest []byte) (string, bool, bool, []error) {
  2468  	request := originalRequest
  2469  	if hasStoredRequest {
  2470  		request = storedRequest
  2471  	}
  2472  
  2473  	accountId, isAppReq, isDOOHReq, err := searchAccountId(request)
  2474  	if err != nil {
  2475  		return "", isAppReq, isDOOHReq, []error{err}
  2476  	}
  2477  
  2478  	// In case the stored request did not have account data we specifically search it in the original request
  2479  	if accountId == "" && hasStoredRequest {
  2480  		accountId, _, _, err = searchAccountId(originalRequest)
  2481  		if err != nil {
  2482  			return "", isAppReq, isDOOHReq, []error{err}
  2483  		}
  2484  	}
  2485  
  2486  	if accountId == "" {
  2487  		return metrics.PublisherUnknown, isAppReq, isDOOHReq, nil
  2488  	}
  2489  
  2490  	return accountId, isAppReq, isDOOHReq, nil
  2491  }
  2492  
  2493  func searchAccountId(request []byte) (string, bool, bool, error) {
  2494  	for _, path := range accountIdSearchPath {
  2495  		accountId, exists, err := getStringValueFromRequest(request, path.key)
  2496  		if err != nil {
  2497  			return "", path.isApp, path.isDOOH, err
  2498  		}
  2499  		if exists {
  2500  			return accountId, path.isApp, path.isDOOH, nil
  2501  		}
  2502  	}
  2503  	return "", false, false, nil
  2504  }
  2505  
  2506  func getStringValueFromRequest(request []byte, key []string) (string, bool, error) {
  2507  	val, dataType, _, err := jsonparser.Get(request, key...)
  2508  	if dataType == jsonparser.NotExist {
  2509  		return "", false, nil
  2510  	}
  2511  	if err != nil {
  2512  		return "", false, err
  2513  	}
  2514  	if dataType != jsonparser.String {
  2515  		return "", true, fmt.Errorf("%s must be a string", strings.Join(key, "."))
  2516  	}
  2517  	return string(val), true, nil
  2518  }
  2519  
  2520  func storedRequestErrorChecker(requestJson []byte, storedRequests map[string]json.RawMessage, storedBidRequestId string) []error {
  2521  	if hasErr, syntaxErr := getJsonSyntaxError(requestJson); hasErr {
  2522  		return []error{fmt.Errorf("Invalid JSON in Incoming Request: %s", syntaxErr)}
  2523  	}
  2524  	if hasErr, syntaxErr := getJsonSyntaxError(storedRequests[storedBidRequestId]); hasErr {
  2525  		return []error{fmt.Errorf("ext.prebid.storedrequest.id refers to Stored Request %s which contains Invalid JSON: %s", storedBidRequestId, syntaxErr)}
  2526  	}
  2527  	return nil
  2528  }
  2529  
  2530  func generateUuidForBidRequest(uuidGenerator uuidutil.UUIDGenerator) ([]byte, error) {
  2531  	newBidRequestID, err := uuidGenerator.Generate()
  2532  	if err != nil {
  2533  		return nil, err
  2534  	}
  2535  	return []byte(`{"id":"` + newBidRequestID + `"}`), nil
  2536  }
  2537  
  2538  func (deps *endpointDeps) setIntegrationType(req *openrtb_ext.RequestWrapper, account *config.Account) error {
  2539  	reqExt, err := req.GetRequestExt()
  2540  	if err != nil {
  2541  		return err
  2542  	}
  2543  	reqPrebid := reqExt.GetPrebid()
  2544  
  2545  	if account == nil || account.DefaultIntegration == "" {
  2546  		return nil
  2547  	}
  2548  	if reqPrebid == nil {
  2549  		reqPrebid = &openrtb_ext.ExtRequestPrebid{Integration: account.DefaultIntegration}
  2550  		reqExt.SetPrebid(reqPrebid)
  2551  	} else if reqPrebid.Integration == "" {
  2552  		reqPrebid.Integration = account.DefaultIntegration
  2553  		reqExt.SetPrebid(reqPrebid)
  2554  	}
  2555  	return nil
  2556  }
  2557  
  2558  func checkIfAppRequest(request []byte) (bool, error) {
  2559  	requestApp, dataType, _, err := jsonparser.Get(request, "app")
  2560  	if dataType == jsonparser.NotExist {
  2561  		return false, nil
  2562  	}
  2563  	if err != nil {
  2564  		return false, err
  2565  	}
  2566  	if requestApp != nil {
  2567  		return true, nil
  2568  	}
  2569  	return false, nil
  2570  }
  2571  
  2572  func (deps *endpointDeps) validateStoredBidRespAndImpExtBidders(prebid *openrtb_ext.ExtImpPrebid, storedBidResp stored_responses.ImpBidderStoredResp, impId string) error {
  2573  	if storedBidResp == nil && len(prebid.StoredBidResponse) == 0 {
  2574  		return nil
  2575  	}
  2576  
  2577  	if storedBidResp == nil {
  2578  		return generateStoredBidResponseValidationError(impId)
  2579  	}
  2580  	if bidResponses, ok := storedBidResp[impId]; ok {
  2581  		if len(bidResponses) != len(prebid.Bidder) {
  2582  			return generateStoredBidResponseValidationError(impId)
  2583  		}
  2584  
  2585  		for bidderName := range bidResponses {
  2586  			if _, bidderNameOk := deps.normalizeBidderName(bidderName); !bidderNameOk {
  2587  				return fmt.Errorf(`unrecognized bidder "%v"`, bidderName)
  2588  			}
  2589  			if _, present := prebid.Bidder[bidderName]; !present {
  2590  				return generateStoredBidResponseValidationError(impId)
  2591  			}
  2592  		}
  2593  	}
  2594  	return nil
  2595  }
  2596  
  2597  func generateStoredBidResponseValidationError(impID string) error {
  2598  	return fmt.Errorf("request validation failed. Stored bid responses are specified for imp %s. Bidders specified in imp.ext should match with bidders specified in imp.ext.prebid.storedbidresponse", impID)
  2599  }