001package com.identityworksllc.iiq.common.service;
002
003import com.identityworksllc.iiq.common.Utilities;
004import org.apache.commons.logging.Log;
005import org.apache.commons.logging.LogFactory;
006import sailpoint.api.IncrementalObjectIterator;
007import sailpoint.api.SailPointContext;
008import sailpoint.object.Application;
009import sailpoint.object.Custom;
010import sailpoint.object.Filter;
011import sailpoint.object.PersistenceOptions;
012import sailpoint.object.QueryOptions;
013import sailpoint.object.Server;
014import sailpoint.object.ServiceDefinition;
015import sailpoint.persistence.HibernatePersistenceManager;
016import sailpoint.server.Service;
017import sailpoint.server.ServicerUtil;
018import sailpoint.tools.GeneralException;
019import sailpoint.tools.Util;
020
021import java.util.Date;
022import java.util.HashMap;
023import java.util.List;
024import java.util.Map;
025import java.util.concurrent.atomic.AtomicLong;
026
027/**
028 * A service to retain the last aggregation timestamps and other cache data
029 * for Applications in a Custom object called "Aggregation Date Storage".
030 *
031 * Since this data is stored in the Application XML itself, it is often inadvertently
032 * overwritten on deployment with an earlier version, or with null data. This will
033 * cause delta tasks to run long or fail entirely. With this service installed,
034 * the Application's dates will be restored as soon as possible after deployment.
035 *
036 * The data will be copied as follows:
037 *
038 * - If there is no data on the Application, but data exists in the Custom object,
039 *   the values on the Application will be replaced by the values in the Custom.
040 *
041 * - If there is no cache for the given Application in the Custom object, the values
042 *   in the Custom will always be set to the Application's values.
043 *
044 * - If the Application's acctAggregationEnd timestamp is newer than the values in the
045 *   Custom object, the Application's values will be copied to the Custom.
046 *
047 * - If the Custom's acctAggregationEnd timestamp is newer than the value in the
048 *   Application object, the Application's values will be replaced by the value
049 *   in the Custom.
050 *
051 * The following attributes are retained in the Custom for each Application:
052 *
053 * - acctAggregationEnd
054 * - acctAggregationStart
055 * - deltaAggregation
056 * - lastAggregationDate
057 *
058 * Three of the four are millisecond timestamps, while deltaAggregation often
059 * contains metadata, like cookies from Azure.
060 */
061public class AggregationDateRetentionService extends Service {
062
063    /**
064     * The field indicating aggregation end (a Date)
065     */
066    public static final String ACCT_AGGREGATION_END = "acctAggregationEnd";
067
068    /**
069     * The field indicating aggregation start (a Date)
070     */
071    public static final String ACCT_AGGREGATION_START = "acctAggregationStart";
072
073    /**
074     * The name of the Custom object storing application aggregation dates
075     */
076    private static final String CUSTOM_NAME = "Aggregation Date Storage";
077
078    /**
079     * The field containing delta aggregation details (e.g., dirSync data in AD)
080     */
081    public static final String DELTA_AGGREGATION = "deltaAggregation";
082
083    /**
084     * The last aggregation date column (used in, e.g., Workday)
085     */
086    public static final String LAST_AGGREGATION_DATE = "lastAggregationDate";
087
088    private static final Log log = LogFactory.getLog(AggregationDateRetentionService.class);
089
090    private final AtomicLong lastEviction = new AtomicLong(0);
091    private final AtomicLong startup = new AtomicLong(0);
092
093    /**
094     * Main entry point for the service. This method is called by the Service. It
095     * will determine if this host is the one that should run the service, and if
096     * so, will call the implementation method.
097     *
098     * @param context SailPoint context
099     * @throws GeneralException if any failures occur
100     */
101    @Override
102    public void execute(SailPointContext context) throws GeneralException {
103        ServiceDefinition self = getDefinition();
104        Server target = null;
105        QueryOptions qo = new QueryOptions();
106        qo.setDirtyRead(false);
107        qo.setCacheResults(false);
108        qo.setCloneResults(true);
109        qo.addOrdering("name", true);
110        if (!Util.nullSafeCaseInsensitiveEq(self.getHosts(), "global")) {
111            qo.addFilter(Filter.in("name", Util.csvToList(self.getHosts())));
112        }
113        List<Server> servers = context.getObjects(Server.class, qo);
114        for(Server s : Util.safeIterable(servers)) {
115            if (!s.isInactive() && ServicerUtil.isServiceAllowedOnServer(context, self, s.getName())) {
116                target = s;
117                break;
118            }
119        }
120        if (target == null) {
121            // This would be VERY strange, since we are, in fact, running the service
122            // right now, in this very code
123            log.warn("There does not appear to be an active server allowed to run service " + self.getName());
124        }
125        String hostname = Util.getHostName();
126        if (target == null || target.getName().equals(hostname)) {
127            if (startup.compareAndSet(0, System.currentTimeMillis())) {
128                log.info("Service " + self.getName() + " first run on host " + hostname + " at " + startup.get());
129            }
130
131            boolean withinTenMinutesOfStartup = System.currentTimeMillis() - startup.get() < 600000;
132            boolean lastClearedOverThreeMinutesAgo = lastEviction.get() == 0 || System.currentTimeMillis() - lastEviction.get() > (60 * 3 * 1000);
133
134            // Within ten minutes of startup, evict on every run (every minute). This handles the
135            // post-deploy case when a new Application object is very likely to be inserted.
136            // Otherwise, evict every three minutes.
137            // There is still a weird overlap period where incremental aggregations may fail.
138            // This is not avoidable using this method.
139            if (withinTenMinutesOfStartup || lastClearedOverThreeMinutesAgo) {
140                HibernatePersistenceManager.getHibernatePersistenceManager(context).clearHighLevelCache();
141                lastEviction.set(System.currentTimeMillis());
142            }
143
144            Utilities.withPrivateContext((privateContext) -> {
145                PersistenceOptions po = new PersistenceOptions();
146                po.setExplicitSaveMode(true);
147
148                privateContext.setPersistenceOptions(po);
149
150                implementation(privateContext);
151            });
152        }
153    }
154
155    /**
156     * Main method for the service, invoked if we are the alphabetically lowest host.
157     *
158     * Queries each application's last aggregation timestamps. If the application's
159     * data is missing or outdated, and there is a retained value, this method restores
160     * the retained value to the Application. If there is no retained value, or if the
161     * retained data is older than the Application data, the retention Custom is updated.
162     *
163     * @param context IIQ context
164     * @throws GeneralException if any failures occur
165     */
166    private void implementation(SailPointContext context) throws GeneralException {
167        Custom custom = context.getObjectByName(Custom.class, CUSTOM_NAME);
168        if (custom == null) {
169            custom = new Custom();
170            custom.setName(CUSTOM_NAME);
171        }
172
173        QueryOptions qo = new QueryOptions();
174        qo.addOrdering("name", true);
175
176        IncrementalObjectIterator<Application> iterator = new IncrementalObjectIterator<>(context, Application.class, qo);
177
178        while(iterator.hasNext()) {
179            Application application = iterator.next();
180
181            boolean updateCustom = false;
182            boolean updateApplication = false;
183
184            Map<String, Object> existingRecord = (Map<String, Object>) custom.get(application.getName());
185            Date lastAggregationEnd = (Date) application.getAttributeValue(ACCT_AGGREGATION_END);
186
187            if (lastAggregationEnd != null) {
188                if (existingRecord != null) {
189                    Date existingLastRunEnd = (Date) existingRecord.get(ACCT_AGGREGATION_END);
190                    if (existingLastRunEnd.before(lastAggregationEnd)) {
191                        updateCustom = true;
192                    } else {
193                        updateApplication = true;
194                    }
195                } else {
196                    updateCustom = true;
197                }
198            } else if (existingRecord != null) {
199                updateApplication = true;
200            }
201
202            if (updateCustom) {
203                if (log.isDebugEnabled()) {
204                    log.debug("Updating Date Retention Custom object for " + application.getName());
205                }
206                Map<String, Object> appData = new HashMap<>();
207                appData.put(ACCT_AGGREGATION_END, application.getAttributeValue(ACCT_AGGREGATION_END));
208                appData.put(ACCT_AGGREGATION_START, application.getAttributeValue(ACCT_AGGREGATION_START));
209                appData.put(DELTA_AGGREGATION, application.getAttributeValue(DELTA_AGGREGATION));
210                appData.put(LAST_AGGREGATION_DATE, application.getAttributeValue(LAST_AGGREGATION_DATE));
211                custom.put(application.getName(), appData);
212            }
213
214            if (updateApplication) {
215                if (log.isDebugEnabled()) {
216                    log.debug("Updating Application data for " + application.getName());
217                }
218                application.setAttribute(ACCT_AGGREGATION_END, existingRecord.get(ACCT_AGGREGATION_END));
219                application.setAttribute(ACCT_AGGREGATION_START, existingRecord.get(ACCT_AGGREGATION_START));
220                application.setAttribute(DELTA_AGGREGATION, existingRecord.get(DELTA_AGGREGATION));
221                application.setAttribute(LAST_AGGREGATION_DATE, existingRecord.get(LAST_AGGREGATION_DATE));
222                context.saveObject(application);
223            }
224        }
225
226        context.saveObject(custom);
227
228        context.commitTransaction();
229    }
230}