1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
use crate::structs::kong::Kong;
use std::{collections::BTreeMap, env};

use regex::Regex;

use semver::Version;

use url::Url;
use uuid::Uuid;

#[allow(unused_imports)] use super::{BaseManifest, ConfigState, Result, Vault};

use super::structs::Authorization;

/// Versioning Scheme used in region
///
/// This is valdiated strictly using `shipcat validate` when versions are found in manifests.
/// Otherwise, it's validated on upgrade time (via `shipcat apply`) when it's passed.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub enum VersionScheme {
    /// Version must be valid semver (no leading v)
    ///
    /// This is the assumed default for regions that lock versions in manifests.
    Semver,
    /// Version must be valid semver or a 40 character hex (git sha)
    ///
    /// This can be used for rolling environments that does not lock versions in manifests.
    GitShaOrSemver,
}

impl Default for VersionScheme {
    fn default() -> VersionScheme {
        VersionScheme::Semver
    }
}

/// Version validator
impl VersionScheme {
    pub fn verify(&self, ver: &str) -> Result<()> {
        let gitre = Regex::new(r"^[0-9a-f\-]{40}$").unwrap();
        match *self {
            VersionScheme::GitShaOrSemver => {
                if !gitre.is_match(&ver) && Version::parse(&ver).is_err() {
                    bail!("Illegal tag {} (floating tags cannot be rolled back please use 40 char git sha or semver)", ver);
                }
            }
            VersionScheme::Semver => {
                if Version::parse(&ver).is_err() {
                    bail!(
                        "Version {} is not a semver version in a region using semver versions",
                        ver
                    );
                }
            }
        };
        Ok(())
    }
}

/// Vault configuration for a region
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(test, derive(Default))]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct VaultConfig {
    /// Vault url up to and including port
    pub url: String,
    /// Root folder under secret/
    ///
    /// Typically, the name of the region to disambiguate.
    pub folder: String,
}

impl VaultConfig {
    pub fn verify(&self, region: &str) -> Result<()> {
        if self.url == "" {
            bail!("Need to set vault url for {}", region);
        }
        if self.folder == "" {
            bail!("Need to set the vault folder for {}", region);
        }
        if self.folder.contains('/') {
            bail!(
                "vault config folder '{}' (under {}) cannot contain slashes",
                self.folder,
                self.url
            );
        }
        Ok(())
    }

    /// Make vault a vault policy for a team based on team ownership
    ///
    /// Returns plaintext hcl
    #[cfg(feature = "filesystem")]
    pub async fn make_policy(&self, mfs: Vec<BaseManifest>, team: &str, env: Environment) -> Result<String> {
        let mut owned_manifests = vec![];
        for mf in mfs {
            if mf.metadata.team == team {
                owned_manifests.push(mf.name);
            }
        }
        let output = self.template(owned_manifests, env).await?;
        Ok(output)
    }
}

//#[derive(Serialize, Deserialize, Clone, Default)]
//#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
// pub struct HostPort {
//    /// Hostname || IP || FQDN
//    pub host: String,
//    /// Port
//    pub port: u32,
//}

/// Kafka configuration for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KafkaConfig {
    /// Broker urls in "hostname:port" format.
    ///
    /// These are injected in to the manifest.kafka struct if it's set.
    pub brokers: Vec<String>,

    /// Proxy urls in "hostname:port" format.
    ///
    /// These are injected in to the manifest.kafka struct if it's set.
    #[serde(default)]
    pub proxies: Vec<String>,

    /// Zookeeper urls in "hostname:port" format.
    ///
    /// These are injected in to the manifest.kafka struct if it's set.
    #[serde(default)]
    pub zk: Vec<String>,

    /// A mapping of kafka properties to environment variables (optional)
    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
    pub propertyEnvMapping: BTreeMap<String, String>,
}

/// Webhook types that shipcat might trigger after actions
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(tag = "name", deny_unknown_fields, rename_all = "snake_case")]
pub enum Webhook {
    /// Audit webhook details
    Audit(AuditWebhook),
}

/// Where / how to send audited events
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct AuditWebhook {
    /// Endpoint
    pub url: Url,
    /// Credential
    pub token: String,
}

/// Configure how CRs will be deployed on a region
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct CRSettings {
    #[serde(rename = "config")]
    pub shipcatConfig: ConfigState,
}

// ----------------------------------------------------------------------------------

/// Kong configuration for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)] // TODO: better Default impl
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KongConfig {
    /// Base URL to use (e.g. uk.dev.babylontech.co.uk)
    pub base_url: String,
    /// Configuration API URL (e.g. https://kong-admin-ops.dev.babylontech.co.uk)
    pub config_url: String,
    /// Kong token expiration time (in seconds)
    pub kong_token_expiration: u32,
    /// TCP logging options
    pub tcp_log: KongTcpLogConfig,
    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
    pub jwt_consumers: BTreeMap<String, KongJwtConsumer>,
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub internal_ips_whitelist: Vec<String>,
    #[serde(default, skip_serializing)]
    pub extra_apis: BTreeMap<String, Kong>,
}

/// StatusCake configuration for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct StatuscakeConfig {
    /// Contact Group that will be used if tests go down
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub contact_group: Option<String>,
    /// Extra tags to add to all tests in this region
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub extra_tags: Option<String>,
}

/// Logz.io configuration for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)] // TODO: better Default impl
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct LogzIoConfig {
    /// Base URL to use (e.g. https://app-eu.logz.io/#/dashboard/kibana/dashboard)
    pub url: String,
    /// Account ID (e.g. 46609)
    pub account_id: String,
}

/// Grafana details for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)] // TODO: better Default impl
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct GrafanaConfig {
    /// Base URL to use (e.g. https://dev-grafana.ops.babylontech.co.uk)
    pub url: String,
    /// Services Dashboard ID (e.g. oHzT4g0iz)
    pub services_dashboard_id: String,
}

/// Sentry details for a region
#[derive(Serialize, Deserialize, Clone, Debug, Default)] // TODO: better Default impl
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct SentryConfig {
    /// Base URL to use (e.g. https://dev-uk-sentry.ops.babylontech.co.uk)
    pub url: String,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KongAnonymousConsumers {
    pub anonymous: BTreeMap<String, String>,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KongOauthConsumer {
    pub oauth_client_id: String,
    pub oauth_client_secret: String,
    pub username: String,
}

#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KongJwtConsumer {
    pub kid: String,
    pub public_key: String,
}

#[derive(Serialize, Deserialize, Clone, Debug, Default)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct KongTcpLogConfig {
    pub enabled: bool,
    pub host: String,
    pub port: String,
}

impl KongConfig {
    pub fn verify(&self) -> Result<()> {
        Ok(())
    }
}

/// Defaults for services in this region
// TODO: This should be ManifestDefaults from shipcat_filebacked
#[derive(Deserialize, Clone, Debug, Default)]
#[serde(default)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct DefaultConfig {
    pub kong: DefaultKongConfig,
}

#[derive(Deserialize, Clone, Debug, Default)]
#[serde(default)]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct DefaultKongConfig {
    pub authorization: Option<Authorization>,
    // HACK: Authorization doesn't have an enabled property, so this allows authorization to be enabled/disabled on a per region basis until we can use AuthorizationSource.
    pub authorizationEnabled: bool,
}

impl Webhook {
    async fn secrets(&mut self, vault: &Vault, region: &str) -> Result<()> {
        match self {
            Webhook::Audit(h) => {
                if h.token == "IN_VAULT" {
                    let vkey = format!("{}/shipcat/WEBHOOK_AUDIT_TOKEN", region);
                    h.token = vault.read(&vkey).await?;
                }
            }
        }
        Ok(())
    }

    async fn verify_secrets_exist(&self, vault: &Vault, region: &str) -> Result<()> {
        match self {
            Webhook::Audit(_h) => {
                let vkey = format!("{}/shipcat/WEBHOOK_AUDIT_TOKEN", region);
                vault.read(&vkey).await?;
            }
        }
        // TODO: when more secrets, build up a list and do a LIST on shipcat folder
        Ok(())
    }

    pub fn get_configuration(&self) -> Result<BTreeMap<String, String>> {
        let mut whc = BTreeMap::default();
        match self {
            Webhook::Audit(_h) => {
                whc.insert(
                    "SHIPCAT_AUDIT_CONTEXT_ID".into(),
                    env::var("SHIPCAT_AUDIT_CONTEXT_ID").unwrap_or_else(|_| Uuid::new_v4().to_string()),
                );

                // if we are on jenkins
                if let (Ok(url), Ok(revision), Ok(_)) = (
                    env::var("BUILD_URL"),
                    env::var("GIT_COMMIT"),
                    env::var("BUILD_NUMBER"),
                ) {
                    whc.insert("SHIPCAT_AUDIT_REVISION".into(), revision);
                    whc.insert("SHIPCAT_AUDIT_CONTEXT_LINK".into(), url);
                }

                // shipcat evars
                if let Ok(url) = env::var("SHIPCAT_AUDIT_CONTEXT_LINK") {
                    whc.insert("SHIPCAT_AUDIT_CONTEXT_LINK".into(), url);
                }
                if let Ok(revision) = env::var("SHIPCAT_AUDIT_REVISION") {
                    whc.insert("SHIPCAT_AUDIT_REVISION".into(), revision);
                }

                // strict requirements
                if !whc.contains_key("SHIPCAT_AUDIT_REVISION") {
                    return Err("SHIPCAT_AUDIT_REVISION not specified".into());
                }

                debug!("Audit webhook config {:?}", whc);
            }
        }

        // TODO: when slack webhook is cfged, require this:
        // slack::have_credentials()?;

        Ok(whc)
    }
}

#[cfg(test)]
mod test_webhooks {
    use super::{AuditWebhook, Webhook};
    use regex::Regex;
    use std::env;
    use url::Url;

    #[test]
    fn region_webhook_audit_config_jenkins_defaults() {
        let wha = Webhook::Audit(AuditWebhook {
            url: Url::parse("http://testnoop").unwrap(),
            token: "noop".into(),
        });
        let reuuid = Regex::new(r"^[0-9a-f-]{36}$").unwrap();

        // enforce jenkins environment
        env::set_var("GIT_COMMIT", "gc1");
        env::set_var("BUILD_URL", "burl");
        env::set_var("BUILD_NUMBER", "1234");

        let cfg = wha.get_configuration().unwrap();

        assert!(reuuid.is_match(&cfg["SHIPCAT_AUDIT_CONTEXT_ID"]));
        assert_eq!(cfg["SHIPCAT_AUDIT_REVISION"], "gc1");
        assert_eq!(cfg["SHIPCAT_AUDIT_CONTEXT_LINK"], "burl");

        // And in serial, test that shipcat-specific evars trumps it
        env::set_var("SHIPCAT_AUDIT_CONTEXT_ID", "cid1");
        env::set_var("SHIPCAT_AUDIT_CONTEXT_LINK", "burl2");
        env::set_var("SHIPCAT_AUDIT_REVISION", "gc2");

        let cfg = wha.get_configuration().unwrap();

        assert_eq!(cfg["SHIPCAT_AUDIT_CONTEXT_ID"], "cid1");
        assert_eq!(cfg["SHIPCAT_AUDIT_CONTEXT_LINK"], "burl2");
        assert_eq!(cfg["SHIPCAT_AUDIT_REVISION"], "gc2");

        // without revision set up, it should err
        env::remove_var("GIT_COMMIT");
        env::remove_var("SHIPCAT_AUDIT_REVISION");

        let cfg = wha.get_configuration();

        assert!(cfg.is_err());
    }
}

// ----------------------------------------------------------------------------------

/// Environments are well defined strings
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[serde(rename_all = "lowercase")]
pub enum Environment {
    /// Production environment
    ///
    /// This environment has limited vault access.
    Prod,

    // Normal environment names
    Preprod,
    Staging,
    Dev,
    Test,

    // Misc environments
    Example,
}

#[cfg(test)]
impl Default for Environment {
    fn default() -> Self {
        Environment::Test
    }
}

impl ToString for Environment {
    fn to_string(&self) -> String {
        // NB: this corresponds to serde serialization atm - used in a few places
        format!("{:?}", self).to_lowercase()
    }
}

// ----------------------------------------------------------------------------------

/// Environments are well defined strings
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
pub enum ReconciliationMode {
    /// Shipcat owned, CRD based decision
    ///
    /// Requires kubernetes 1.13 and above (default).
    /// If CRD was configured, kube apply chart with owner references
    CrdOwned,
}

impl Default for ReconciliationMode {
    fn default() -> Self {
        ReconciliationMode::CrdOwned
    }
}

// ----------------------------------------------------------------------------------

/// A region is an abstract kube context
///
/// Either it's a pure kubernetes context with a namespace and a cluster,
/// or it's an abstract concept with many associated real kubernetes contexts.
#[derive(Serialize, Deserialize, Debug, Clone)]
#[cfg_attr(test, derive(Default))]
#[cfg_attr(feature = "filesystem", serde(deny_unknown_fields))]
pub struct Region {
    /// Name of region
    pub name: String,
    /// Kubernetes namespace
    pub namespace: String,
    /// Environment (e.g. `dev` or `staging`)
    pub environment: Environment,
    /// Reconciliation mode
    ///
    /// This affects how `cluster crd reconcile` behaves in the region.
    #[serde(default)]
    pub reconciliationMode: ReconciliationMode,

    /// Primary cluster serving this region
    ///
    /// Shipcat does not use this for to decide where a region gets deployed,
    /// but it is used to indicate where the canonical location of a cluster is.
    ///
    /// During blue/green cluster failovers the value of this string may not be accurate.
    ///
    /// Jobs that decide where to deploy a region to should use `get clusterinfo`
    /// with explicit cluster names and regions.
    pub cluster: String,
    /// Versioning scheme
    pub versioningScheme: VersionScheme,

    /// Important base urls that can be templated in evars
    #[serde(default)]
    pub base_urls: BTreeMap<String, String>,

    /// Kong configuration for the region
    #[serde(default)]
    pub kong: Option<KongConfig>,
    /// Statuscake configuration for the region
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub statuscake: Option<StatuscakeConfig>,
    /// List of Whitelisted IPs
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub ip_whitelist: Vec<String>,
    /// Kafka configuration for the region
    #[serde(default)]
    pub kafka: KafkaConfig,
    /// Vault configuration for the region
    pub vault: VaultConfig,
    /// Logz.io configuration for the region
    pub logzio: Option<LogzIoConfig>,
    /// Grafana details for the region
    pub grafana: Option<GrafanaConfig>,
    /// Sentry URL for the region
    pub sentry: Option<SentryConfig>,
    /// List of locations the region serves
    #[serde(default)]
    pub locations: Vec<String>,
    /// All webhooks
    #[serde(default, skip_serializing_if = "Vec::is_empty")]
    pub webhooks: Vec<Webhook>,
    /// CRD tuning
    pub customResources: Option<CRSettings>,

    /// Old default values for services
    // TODO: Remove after everything has been migrated to `defaultsV2`
    #[serde(skip_serializing, default)]
    pub defaults: Option<DefaultConfig>,
    /// Old default environment variables to inject
    // TODO: Remove after everything has been migrated to `defaultsV2`
    #[serde(default, skip_serializing)]
    pub env: Option<BTreeMap<String, String>>,
    /// Default values for services (used by shipcat_filebacked only)
    // TODO: Rename to `defaults` after removing legacy field
    #[serde(skip_serializing, default)]
    #[cfg(feature = "filesystem")]
    pub defaultsV2: Option<serde_yaml::Value>,

    /// The regular expression used to verify destination rules' regions
    #[serde(default, skip_serializing_if = "Option::is_none", with = "serde_regex")]
    pub destinationRuleHostRegex: Option<Regex>,
}

impl Region {
    // Internal secret populator for Config::new
    pub async fn secrets(&mut self) -> Result<()> {
        let v = Vault::regional(&self.vault)?;
        for wh in self.webhooks.iter_mut() {
            wh.secrets(&v, &self.name).await?;
        }
        Ok(())
    }

    // Entry point for region verifier
    pub async fn verify_secrets_exist(&self) -> Result<()> {
        let v = Vault::regional(&self.vault)?;
        for wh in &self.webhooks {
            wh.verify_secrets_exist(&v, &self.name).await?;
        }
        Ok(())
    }

    // Get the Vault URL for a given service in this region
    pub fn vault_url(&self, app: &str) -> String {
        let vault_url = self.vault.url.clone();
        let path = "/ui/vault/secrets/secret/list/";
        format!(
            "{vault_url}/{path}/{env}/{app}/",
            vault_url = vault_url.trim_matches('/'),
            path = path.trim_matches('/'),
            env = &self.name,
            app = &app
        )
    }

    pub fn grafana_url(&self, app: &str) -> Option<String> {
        self.grafana.clone().map(|gf| {
            format!("{grafana_url}/d/{dashboard_id}/kubernetes-services?var-cluster={cluster}&var-namespace={namespace}&var-deployment={app}",
              grafana_url = gf.url.trim_matches('/'),
              dashboard_id = gf.services_dashboard_id,
              app = app,
              cluster = &self.cluster,
              namespace = &self.namespace)
        })
    }

    // Get the Sentry URL for a given service slug in a cluster in this region
    pub fn sentry_url(&self, slug: &str) -> Option<String> {
        self.sentry.clone().map(|s| {
            format!(
                "{sentry_base_url}/sentry/{slug}",
                sentry_base_url = s.url,
                slug = slug
            )
        })
    }

    pub fn logzio_url(&self, app: &str) -> Option<String> {
        self.logzio.clone().map(|lio| {
            format!(
                "{logzio_url}/{app}-{env}?&switchToAccountId={account_id}",
                logzio_url = lio.url.trim_matches('/'),
                app = app,
                env = self.name,
                account_id = lio.account_id
            )
        })
    }

    pub fn raftcat_url(&self) -> Option<String> {
        let devops = String::from("dev-ops");
        let region_name = env::var("REGION_NAME").ok()?;
        if region_name == devops {
            Some(String::from("https://raftcat.ops.babylontech.co.uk/raftcat/"))
        } else {
            self.base_urls
                .get("external_services")
                .map(|base| format!("{base}/raftcat/", base = base))
        }
    }
}