Introduction

The idea behind this proposal is to simplify the configuration files and launch sequence process of RTBKit.

Why

Currently, we have many configuration files :

• launch_sequence.json: describes the topology of the installation, what services to launch for the current node ;
• router_config.json: the router configuration file (specifies what type of exchange connector, auction endpoint, ...) ;
• exchange.json: the exchange configuration file (type of bids, ... ) ; adserver-config.json: the Adserver configuration (type of adserver connector, ..) ;
• bootstrap.json: common configuration file for various components (ZooKeeper, carbon, port ranges. ...).

Which makes a total of 5 configuration files to configure the whole stack. Moreover, the launch sequence configuration file could be simplified. Switching between configuration files is sometimes less convenient rather than searching for a configuration element in a single configuration file.

Proposal

Step 1: Simplified launch sequence

This first step of this proposal consists of simplifying the launch_sequence configuration file. Below is an example of the new version of this file, initially inspired by Rémi :

{
 
// [ config ]===================================================================
 
    "root": "/path/to/root/rtb",
 
// [ services ]=================================================================
 
    "services": {
        "monitor":            "./build/x86_64/bin/monitor_service_runner",
        "agentConfiguration": "./build/x86_64/bin/agent_configuration_service_runner",
        "adServer":           "./build/x86_64/bin/dummy_ad_server_connector_runner",
        "router":             "./build/x86_64/bin/router_runner",
 
        "banker": {
            "exec": "./build/x86_64/bin/banker_service_runner",
            "args": { "r": "localhost:1234" }
        },
 
        "logger": {
            "exec": "./build/x86_64/bin/cloud_logger_runner",
            "args": {
                "key-id": "ACCESSKEYID",
                "key": "YOUSHALLNOTPASS",
                "bucket": "end.of.the.world",
                "prefix": "logger",
                "log-dir": "/logs/meow",
                "cloud-dir": "/logs/cloud-backup/"
            }
        },
 
        "agent-monitor": {
            "exec": "./build/x86_64/bin/agent_monitor",
            "args": { "%carbonPrefix": "%n.agents" }
        },

        "mock-client": {
            "exec": "./path/to/installation/local/rtb/rtb_virtualenv/bin/python",
            "args": {
                "./build/x86_64/bin/mock_client": null,
                "--sleep": 3600,
                "--freq": 0.5
            }
        }
    },
 
 
// [ nodes ]====================================================================
 
    "nodes": {
 
        "foo.rtb1": {
            "bootstrap": "bootstrap.foo.json",
            "tasks": [
                "banker",
                "monitor",
                "agentConfiguration",
                "adServer",
                "logger"
            ]
        },
 
        "foo.rtb2": {
            "bootstrap": "bootstrap.foo.json",
            "tasks": [ "router", "agent-monitor" ]
        },
 
 
        "foo.rtb3": {
            "bootstrap": "bootstrap.foo.json",
            "tasks": [ "router", "agent-monitor" ]
        },
 
        "foo.rtb4": {
            "bootstrap": "bootstrap.foo.json",
            "tasks": [ "router", "agent-monitor" ]
        },
 
 
        "bar.rtb1": {
            "bootstrap": "bootstrap.bar.json",
            "tasks": [
                "monitor",
                "agentConfiguration",
                "adServer",
                "logger"
            ]
        },
 
        "bar.rtb2": {
            "bootstrap": "bootstrap.bar.json",
            "tasks": [ "router", "agent-monitor" ]
        },
 
        "bar.rtb3": {
            "bootstrap": "bootstrap.bar.json",
            "tasks": [ "router", "agent-monitor" ]
        },
 
        "bar.rtb4": {
            "bootstrap": "bootstrap.bar.json",
            "tasks": [ "router", "agent-monitor" ]
        }
    }
}

Step2: Unifying bootstrap.json and other

The goal is to regroup all the config files (bootstrap, adserver, router and exchange) into one single config file, simpler to edit and reason about.

{
     "installation": "rtb",
     "location": "foo",

     "zookeeper-uri": "zookeeper-host:4292",
     "carbon-uri": "carbon-host:3009",

     "bidderApiHost":"https://bidder-api-host.com",
     "secret_token":"aaAAaaABbBbBbBbCccCcCdddDDdeeEeeeEeFFfFffFFF",
     "s3AccessKeyId":"ACCESSKEYID",
     "s3AccessKey":"ACCESSKEY",
     "bankerHost":"http://banker-host:9876",

     "portRanges": {
         "logs":                   [16000, 17000],
         "router":                 [17000, 18000],
         "augmentors":             [18000, 19000],
         "configuration":          [19000, 20000],
         "postAuctionLoop":        [20000, 21000],
         "postAuctionLoopAgents":  [21000, 22000],

         "banker.zmq":             [22000, 23000],
         "banker.http":              9876,

         "agentConfiguration.zmq": [23000, 24000],
         "agentConfiguration.http":  9986,

         "monitor.zmq":            [24000, 25000],
         "monitor.http":             9987,

         "adServer.logger":        [25000, 26000],

         "configValidator.zmq":    [36000, 37000],
         "configValidator.http":     9005,

         "availability.zmq":       [34000, 35000],
         "availability.http":        9981,

         "globalStatsTable.zmq":   [33000, 34000],
         "globalStatsTable.http":    9983,

         "blacklist.zmq":          [35000, 36000],
         "blacklist.http":           9984,

     },

    "router": {
          "exchangeType": "openrtb",
          "numThreads": 4,
          "listenPort": 9950,
          "bindHost": "0.0.0.0",
          "auctionVerb": "POST",
          "auctionResource": "/auctions",
          "performNameLookup": false
      },

    "adserver": {
          "type": "standard",
          "winPort": 54321,
          "eventsPort": 42135
      },
      "exchange": {
          "workers": [
           {
              "threads": 50,
              "bids": {
                  "type": "openrtb",
                  "url": "foobar.com:1234",
                  "resource": "/auctions"
               },
               "wins": {
                   "type": "standard",
                   "url": "foobar.com:54321"
               }
            }
        ]
    }
}