Skip to content

Commit

Permalink
feat(cats): Adds cats scheduler from Clouddriver, which will eventual…
Browse files Browse the repository at this point in the history
…ly eliminate the need for fiat.writeMode.enabled (#185)
  • Loading branch information
Travis Tomsu authored Jun 21, 2017
1 parent 2dfa632 commit e775f0d
Show file tree
Hide file tree
Showing 13 changed files with 162 additions and 76 deletions.
2 changes: 2 additions & 0 deletions fiat-roles/fiat-roles.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ dependencies {
compile spinnaker.dependency("bootActuator")
compile spinnaker.dependency("bootWeb")
compile spinnaker.dependency("korkHystrix")
// TODO(ttomsu): expose this in spinnaker-dependencies
compile "com.netflix.spinnaker.clouddriver:cats-redis:1.643.0"

compile "redis.clients:jedis:2.6.2"
compile "com.google.api-client:google-api-client:1.21.0"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Copyright 2017 Netflix, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License")
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.netflix.spinnaker.fiat.config;

import com.netflix.spinnaker.cats.agent.Agent;
import com.netflix.spinnaker.cats.agent.ExecutionInstrumentation;
import com.netflix.spinnaker.cats.redis.JedisSource;
import com.netflix.spinnaker.cats.redis.cluster.AgentIntervalProvider;
import com.netflix.spinnaker.cats.redis.cluster.ClusteredAgentScheduler;
import com.netflix.spinnaker.cats.redis.cluster.DefaultAgentIntervalProvider;
import com.netflix.spinnaker.cats.redis.cluster.DefaultNodeIdentity;
import com.netflix.spinnaker.cats.redis.cluster.DefaultNodeStatusProvider;
import com.netflix.spinnaker.cats.redis.cluster.NodeIdentity;
import com.netflix.spinnaker.cats.redis.cluster.NodeStatusProvider;
import com.netflix.spinnaker.fiat.roles.UserRolesSyncer;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;

import javax.annotation.PreDestroy;
import java.util.concurrent.TimeUnit;

@Slf4j
@Configuration
@ConditionalOnExpression("${fiat.writeMode.enabled:true}")
public class CatsSchedulerConfig {

@Autowired
UserRolesSyncer userRolesSyncer;

ClusteredAgentScheduler scheduler;

@Value("${fiat.writeMode.syncDelayMs:600000}")
String syncDelayMs;

@Bean
NodeIdentity nodeIdentity() {
return new DefaultNodeIdentity();
}

@Bean
AgentIntervalProvider agentIntervalProvider() {
return new DefaultAgentIntervalProvider(Long.parseLong(syncDelayMs), 10000);
}

@Bean
NodeStatusProvider nodeStatusProvider() {
return new DefaultNodeStatusProvider();
}

@Bean
ClusteredAgentScheduler clusteredAgentScheduler(JedisSource jedisSource,
NodeIdentity nodeIdentity,
AgentIntervalProvider intervalProvider,
NodeStatusProvider nodeStatusProvider,
ExecutionInstrumentation executionInstrumentation) {
scheduler = new ClusteredAgentScheduler(jedisSource, nodeIdentity, intervalProvider, nodeStatusProvider);
scheduler.schedule(userRolesSyncer, userRolesSyncer.getAgentExecution(null), executionInstrumentation);
return scheduler;
}

@PreDestroy
void cleanup() {
scheduler.unschedule(userRolesSyncer);
}

@Slf4j
@Component
static class LoggingInstrumentation implements ExecutionInstrumentation {

@Override
public void executionStarted(Agent agent) {
log.debug("{}:{} starting", agent.getProviderName(), agent.getAgentType());
}

@Override
public void executionCompleted(Agent agent, long durationMs) {
log.info("{}:{} completed in {}s", agent.getProviderName(), agent.getAgentType(), TimeUnit.MILLISECONDS.toSeconds(durationMs));
}

@Override
public void executionFailed(Agent agent, Throwable cause) {
log.warn(agent.getAgentType() + ":" + agent.getAgentType() + " failed", cause);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package com.netflix.spinnaker.fiat.config;

import com.netflix.spinnaker.fiat.redis.JedisPoolSource;
import com.netflix.spinnaker.fiat.redis.JedisSource;
import com.netflix.spinnaker.cats.redis.JedisPoolSource;
import com.netflix.spinnaker.cats.redis.JedisSource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
import com.google.common.collect.ArrayTable;
import com.google.common.collect.HashBasedTable;
import com.google.common.collect.Table;
import com.netflix.spinnaker.cats.redis.JedisSource;
import com.netflix.spinnaker.fiat.config.UnrestrictedResourceConfig;
import com.netflix.spinnaker.fiat.model.UserPermission;
import com.netflix.spinnaker.fiat.model.resources.Resource;
import com.netflix.spinnaker.fiat.model.resources.ResourceType;
import com.netflix.spinnaker.fiat.model.resources.Role;
import com.netflix.spinnaker.fiat.redis.JedisSource;
import lombok.NonNull;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -39,7 +39,6 @@
import redis.clients.jedis.Pipeline;
import redis.clients.jedis.Response;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.netflix.spinnaker.fiat.providers.ProviderHealthTracker;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.List;
Expand All @@ -34,7 +35,7 @@
* health tracker, which will turn unhealthy after X number of failed cache refreshes.
*/
@Slf4j
public class ClouddriverService implements HealthTrackable {
public class ClouddriverService implements HealthTrackable, InitializingBean {

private static final String GROUP_KEY = "clouddriverService";

Expand All @@ -51,6 +52,16 @@ public ClouddriverService(ClouddriverApi clouddriverApi) {
this.clouddriverApi = clouddriverApi;
}

@Override
public void afterPropertiesSet() throws Exception {
try {
getAccounts();
getApplications();
} catch (Exception e) {
log.warn("Cache initialization failed: ", e);
}
}

public List<Account> getAccounts() {
return new SimpleJava8HystrixCommand<>(
GROUP_KEY,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,14 @@
import com.netflix.spinnaker.fiat.providers.ProviderHealthTracker;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Autowired;

import java.util.List;
import java.util.concurrent.atomic.AtomicReference;

@Slf4j
public class Front50Service implements HealthTrackable {
public class Front50Service implements HealthTrackable, InitializingBean {

private static final String GROUP_KEY = "front50Service";

Expand All @@ -46,6 +47,17 @@ public Front50Service(Front50Api front50Api) {
this.front50Api = front50Api;
}

@Override
public void afterPropertiesSet() throws Exception {
try {
// Initialize caches (also indicates service is healthy)
getAllApplicationPermissions();
getAllServiceAccounts();
} catch (Exception e) {
log.warn("Cache prime failed: ", e);
}
}

public List<Application> getAllApplicationPermissions() {
return new SimpleJava8HystrixCommand<>(
GROUP_KEY,
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.netflix.spinnaker.fiat.roles;

import com.diffplug.common.base.Functions;
import com.netflix.spinnaker.cats.agent.RunnableAgent;
import com.netflix.spinnaker.fiat.config.ResourceProvidersHealthIndicator;
import com.netflix.spinnaker.fiat.config.UnrestrictedResourceConfig;
import com.netflix.spinnaker.fiat.model.UserPermission;
Expand All @@ -28,6 +29,7 @@
import com.netflix.spinnaker.fiat.permissions.PermissionsResolver;
import com.netflix.spinnaker.fiat.providers.ProviderException;
import com.netflix.spinnaker.fiat.providers.ResourceProvider;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
Expand All @@ -47,7 +49,12 @@
@Slf4j
@Component
@ConditionalOnExpression("${fiat.writeMode.enabled:true}")
public class UserRolesSyncer {
public class UserRolesSyncer implements RunnableAgent {

@Getter
private final String agentType = "UserRoleSyncer";
@Getter
private final String providerName = "Fiat";

@Autowired
@Setter
Expand All @@ -69,8 +76,7 @@ public class UserRolesSyncer {
@Setter
private long retryIntervalMs;

@Scheduled(fixedDelayString = "${fiat.writeMode.syncDelayMs:600000}")
public void sync() {
public void run() {
syncAndReturn();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ package com.netflix.spinnaker.fiat.permissions

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.ObjectMapper
import com.netflix.spinnaker.cats.redis.JedisSource
import com.netflix.spinnaker.fiat.config.UnrestrictedResourceConfig
import com.netflix.spinnaker.fiat.model.UserPermission
import com.netflix.spinnaker.fiat.model.resources.Account
import com.netflix.spinnaker.fiat.model.resources.Application
import com.netflix.spinnaker.fiat.model.resources.Role
import com.netflix.spinnaker.fiat.model.resources.ServiceAccount
import com.netflix.spinnaker.fiat.redis.JedisSource
import com.netflix.spinnaker.kork.jedis.EmbeddedRedis
import redis.clients.jedis.Jedis
import spock.lang.AutoCleanup
Expand Down Expand Up @@ -277,26 +277,26 @@ class RedisPermissionsRepositorySpec extends Specification {
def result = repo.getAllByRoles(["role1"])

then:
result == ["user1": user1.merge(unrestricted),
"user2": user2.merge(unrestricted),
result == ["user1" : user1.merge(unrestricted),
"user2" : user2.merge(unrestricted),
(UNRESTRICTED): unrestricted]

when:
result = repo.getAllByRoles(["role3", "role4"])

then:
result == ["user2": user2.merge(unrestricted),
"user4": user4.merge(unrestricted),
result == ["user2" : user2.merge(unrestricted),
"user4" : user4.merge(unrestricted),
(UNRESTRICTED): unrestricted];

when:
result = repo.getAllByRoles(null);

then:
result == ["user1": user1.merge(unrestricted),
"user2": user2.merge(unrestricted),
"user3": user3.merge(unrestricted),
"user4": user4.merge(unrestricted),
result == ["user1" : user1.merge(unrestricted),
"user2" : user2.merge(unrestricted),
"user3" : user3.merge(unrestricted),
"user4" : user4.merge(unrestricted),
(UNRESTRICTED): unrestricted]

when:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package com.netflix.spinnaker.fiat.roles

import com.fasterxml.jackson.annotation.JsonInclude
import com.fasterxml.jackson.databind.ObjectMapper
import com.netflix.spinnaker.cats.redis.JedisSource
import com.netflix.spinnaker.fiat.config.ResourceProvidersHealthIndicator
import com.netflix.spinnaker.fiat.config.UnrestrictedResourceConfig
import com.netflix.spinnaker.fiat.model.UserPermission
Expand All @@ -27,7 +28,6 @@ import com.netflix.spinnaker.fiat.model.resources.ServiceAccount
import com.netflix.spinnaker.fiat.permissions.PermissionsResolver
import com.netflix.spinnaker.fiat.permissions.RedisPermissionsRepository
import com.netflix.spinnaker.fiat.providers.ResourceProvider
import com.netflix.spinnaker.fiat.redis.JedisSource
import com.netflix.spinnaker.kork.jedis.EmbeddedRedis
import org.springframework.boot.actuate.health.Health
import redis.clients.jedis.Jedis
Expand Down
Loading

0 comments on commit e775f0d

Please sign in to comment.