- * Filters the contents of an unzipped feed file - *
- * - * @param filterRequest+ * Filters the contents of an unzipped feed file + *
+ * + * @param filterRequest- * Unzip downloaded feed file and create a new file - *
- * - * @param filePath The absolute path to the gzipped feed file - * @return Response Response - */ - public Response unzip(String filePath); + /** + *+ * Unzip downloaded feed file and create a new file + *
+ * + * @param filePath The absolute path to the gzipped feed file + * @return Response Response + */ + public Response unzip(String filePath); + /** + *+ * Downloads the feed file as specified by the parameters in the request. + * Based on the feedRequest values, if a file is available, then it is + * downloaded and the method returns the file path. + * + * If the file is not available, then the error details are returned. + *
+ * + * @param feedRequest Container for capturing parameters for downloading + * feed file + * @param downloadDirectory The local directory where the file should be + * downloaded. Default location is the current working directory + * @return GetFeedResponse GetFeedResponse + */ + public GetFeedResponse get(FeedRequest feedRequest, String downloadDirectory); - /** - *- * Downloads the feed file as specified by the parameters in the request. Based on the feedRequest - * values, if a file is available, then it is downloaded and the method returns the file path. - * - * If the file is not available, then the error details are returned. - *
- * - * @param feedRequest Container for capturing parameters for downloading feed file - * @param downloadDirectory The local directory where the file should be downloaded. Default location is - * the current working directory - * @return GetFeedResponse GetFeedResponse - */ - public GetFeedResponse get(FeedRequest feedRequest, String downloadDirectory); + /** + *+ * Downloads the feed file as specified by the parameters in the + * request.Based on the feedRequest values, if a file is available, then it + * is downloaded and the method returns the file path. If the file is not + * available, then the error details are returned. + *
+ * + * @param feedRequest Container for capturing parameters for downloading + * feed file + * @param downloadDirectory The local directory where the file should be + * downloaded. Default location is the current working directory + * @param credentialConfigFilePath credentials file absolute path, required + * for token refresh on the fly to support resume capability. Default + * location is the current working directory + * @return GetFeedResponse GetFeedResponse + */ + public GetFeedResponse get(FeedRequest feedRequest, String downloadDirectory, String credentialConfigFilePath); - /** - *
- * Given a config file, this method parses the input, and performs necessary actions,
- * as specified in the config file.
- *
- * The config file should consist of json and follow the structure :-
- * {
- *
- * "requests": [
- * {
- * "feedRequest": {
- *
- * },
- * "filterRequest": {
- *
- * }
- * }
- *
- * ]
- * }
- *
- *
- * If the original file is already downloaded, then the feedRequest becomes optional,
- * and the unzipped file path can be provided in the 'inputFilePath' field in the
- * filter request,
- * along with the other filter characteristics.
- *
- * If the entire workflow needs to be performed i.e - download, unzip and filter,
- * then both feedRequest and filterRequest are required.
- * The inputFilePath can be omitted in this case.
- *
- *
- * @param configFile The absolute path of the file, which consists of a list of feed request or filter
- * request or both
- * @param token The oAuth token with the appropriate scope Eg - Bearer vxxx
- * @return List of type Response
- */
- public List
+ * Given a config file, this method parses the input, and performs necessary actions,
+ * as specified in the config file.
+ *
+ * The config file should consist of json and follow the structure :-
+ * {
+ *
+ * "requests": [
+ * {
+ * "feedRequest": {
+ *
+ * },
+ * "filterRequest": {
+ *
+ * }
+ * }
+ *
+ * ]
+ * }
+ *
+ *
+ * If the original file is already downloaded, then the feedRequest becomes optional,
+ * and the unzipped file path can be provided in the 'inputFilePath' field in the
+ * filter request,
+ * along with the other filter characteristics.
+ *
+ * If the entire workflow needs to be performed i.e - download, unzip and filter,
+ * then both feedRequest and filterRequest are required.
+ * The inputFilePath can be omitted in this case.
+ *
+ *
+ * @param configFile The absolute path of the file, which consists of a list
+ * of feed request or filter request or both
+ * @param token The oAuth token with the appropriate scope Eg - Bearer vxxx
+ * @return List of type Response
+ */
+ public List+ * Downloads the feed file as specified by the parameters in the + * request.Based on the feedRequest values, if a file is available, then it + * is downloaded and the method returns the file path. If the file is not + * available, then the error details are returned. + *
+ * + * @param feedRequest Container for capturing parameters for downloading + * feed file + * @param downloadDirectory The local directory where the file should be + * downloaded. Default location is the current working directory + * @param credentialConfigFilePath credentials file absolute path, required + * for token refresh on the fly to support resume capability while + * downloading feeds. Default location is the current working directory + * @return GetFeedResponse GetFeedResponse + */ + @Override + public GetFeedResponse get(FeedRequest feedRequest, String downloadDirectory, String credentialConfigFilePath) { + + if (null == credentialConfigFilePath || credentialConfigFilePath.isEmpty()) { + LOGGER.debug("********* using default credentialFilePath = {}", credentialFilePath); + credentialConfigFilePath = credentialFilePath; + } + LOGGER.debug("********* Begin Feed.get()"); - if (feedRequest == null) { - return new GetFeedResponse(-1, "feedRequest is null.Aborting..", null, null); - } + GetFeedResponse response = null; + Path downloadDirectoryPath = null; - // if null, then take default path - current directory - if (downloadDirectory == null) { - downloadDirectoryPath = Paths.get(".").toAbsolutePath().normalize(); - } else { - downloadDirectoryPath = Paths.get(downloadDirectory); - } + if (feedRequest == null) { + return new GetFeedResponse(-1, "feedRequest is null.Aborting..", null, null); + } - LOGGER.debug("feedRequest params = {}", feedRequest.toString()); + // if null, then take default path - current directory + if (downloadDirectory == null) { + downloadDirectoryPath = Paths.get(".").toAbsolutePath().normalize(); + } else { + downloadDirectoryPath = Paths.get(downloadDirectory); + } - try { - response = process(feedRequest, downloadDirectoryPath); + LOGGER.debug("feedRequest params = {}", feedRequest.toString()); - } catch (Exception e) { - LOGGER.error("Exception in Feed.get()", e); - e.printStackTrace(); - response = new GetFeedResponse(Constants.FAILURE_CODE, e.getMessage(), null, null); - } + try { + response = process(feedRequest, downloadDirectoryPath, credentialConfigFilePath); - LOGGER.debug("Exiting Feed.get() {}", response.getFilePath()); - return response; - } + } catch (Exception e) { + LOGGER.error("Exception in Feed.get()", e); + e.printStackTrace(); + response = new GetFeedResponse(Constants.FAILURE_CODE, e.getMessage(), null, null); + } - /** - *- * The process method performs pre requisite functions, before calling the feed API - Adds - * additional headers including range - Creates default file/folder path and cleansup if already - * present - *
- * - * @param feedRequest - * @param downloadDirectory Optional local directory where files can be downloaded. Default is - * current working directory - * @return - * @throws IOException - */ - private GetFeedResponse process(FeedRequest feedRequest, Path downloadDirectory) - throws IOException { - - LOGGER.debug("Entering Feed.process()"); - - if (feedRequest.getToken() == null || feedRequest.getType() == null - || feedRequest.getCategoryId() == null || feedRequest.getMarketplaceId() == null) { - return new GetFeedResponse(-1, "Mandatory feedRequest parameters are null. Aborting..", null, - null); + LOGGER.debug("Exiting Feed.get() {}", response.getFilePath()); + return response; } - Request.Builder requestBuilder = new Request.Builder(); + /** + *+ * The process method performs pre requisite functions, before calling the + * feed API - Adds additional headers including range - Creates default + * file/folder path and cleansup if already present + *
+ * + * @param feedRequest + * @param downloadDirectory Optional local directory where files can be + * downloaded. Default is current working directory + * @return + * @throws IOException + */ + private GetFeedResponse process(FeedRequest feedRequest, Path downloadDirectory, String credentialConfigFilePath) + throws IOException { + + LOGGER.debug("Entering Feed.process()"); + + if (feedRequest.getToken() == null || feedRequest.getType() == null + || feedRequest.getCategoryId() == null || feedRequest.getMarketplaceId() == null) { + return new GetFeedResponse(-1, "Mandatory feedRequest parameters are null. Aborting..", null, + null); + } - Path pathToFile = - Paths.get(downloadDirectory.toString() + "/" + feedUtils.generateFileName(feedRequest)); - LOGGER.debug("Path to store file = {}", pathToFile); + // checking if date format is valid and date is within valid range + if (!feedValidator.isValidDateFormatAndRange(feedRequest)) { + LOGGER.debug("Date format is not valid or Date is not within valid range. Cannot get feed. Aborting..."); + return new GetFeedResponse(-1, "Date format is not valid or Date is not within valid range. Cannot get feed. Aborting...", + null, null); + } - if (Files.exists(pathToFile)) { - Files.delete(pathToFile); - } else { - Files.createDirectories(pathToFile.getParent()); - } + Request.Builder requestBuilder = new Request.Builder(); - Path path = Files.createFile(pathToFile); + Path pathToFile + = Paths.get(downloadDirectory.toString() + "/" + feedUtils.generateFileName(feedRequest)); + LOGGER.debug("Path to store file = {}", pathToFile); - // generate static request - requestBuilder = feedUtils.generateRequest(feedRequest, requestBuilder); + if (Files.exists(pathToFile)) { + Files.delete(pathToFile); + } else { + Files.createDirectories(pathToFile.getParent()); + } - // generate dynamic header - Long chunkSizeLimit = feedUtils.getChunkSizeLimit(feedRequest); - requestBuilder.addHeader(Constants.RANGE_HEADER, Constants.RANGE_PREFIX + chunkSizeLimit); + Path path = Files.createFile(pathToFile); - // invoke request - return invoker(requestBuilder, path, true, chunkSizeLimit); - } + // generate static request + requestBuilder = feedUtils.generateRequest(feedRequest, requestBuilder); - /** - *- * Invokes the feed API with the max range value of 100 MB. If the file is lesser than 100 MB, - * then it returns the downloaded file path along with the status. - * - * If the file is greater than 100 MB - Iteratively calls feed API, with incrementing range - * headers - Appends content to file - Downloads entire content and returns with downloaded file - * path. - * - *
- * - * @param request The API request - * @param path Path of the downloaded or partially downloading file, where contents need to be - * appended - * @param isStart - THe first request sets this parameter to true. If subsequent requests are - * required in case of 206, then this is set to false. - * @param chunkSizeLimit - This indicates the max chunkSize limit. For prod, it is 100 MB and for sandbox, it is 10MB. - * @return - */ - private GetFeedResponse invoker(Request.Builder requestBuilder, Path path, boolean isStart, Long chunkSizeLimit) { + // generate dynamic header + Long chunkSizeLimit = feedUtils.getChunkSizeLimit(feedRequest); + requestBuilder.addHeader(Constants.RANGE_HEADER, Constants.RANGE_PREFIX + chunkSizeLimit); - LOGGER.debug("Entering Feed.invoker()"); + // invoke request + return invoker(requestBuilder, path, true, chunkSizeLimit, feedRequest, credentialConfigFilePath); + } - Request request = requestBuilder.build(); + /** + *+ * Invokes the feed API with the max range value of 100 MB. If the file is + * lesser than 100 MB, then it returns the downloaded file path along with + * the status. + * + * If the file is greater than 100 MB - Iteratively calls feed API, with + * incrementing range headers - Appends content to file - Downloads entire + * content and returns with downloaded file path. + * + *
+ * + * @param request The API request + * @param path Path of the downloaded or partially downloading file, where + * contents need to be appended + * @param isStart - THe first request sets this parameter to true. If + * subsequent requests are required in case of 206, then this is set to + * false. + * @param chunkSizeLimit - This indicates the max chunkSize limit. For prod, + * it is 100 MB and for sandbox, it is 10MB. + * @return + */ + private GetFeedResponse invoker(Request.Builder requestBuilder, Path path, boolean isStart, Long chunkSizeLimit, FeedRequest feedRequest, String credentialConfigFilePath) { + + LOGGER.debug("Entering Feed.invoker()"); + + Request request = requestBuilder.build(); + + LOGGER.debug("API request = {}", request.toString()); + + InvokeResponse responseFlag = invokeIteratively(request, path); + + LOGGER.debug("First API Response = {}", responseFlag.toString()); + + if (responseFlag.getStatusCode() == 200) { + LOGGER.debug("First API Response is 200. All done.."); + return new GetFeedResponse(Constants.SUCCESS_CODE, Constants.SUCCESS, fixFilePath(path, responseFlag), null); + + } else if (responseFlag.getStatusCode() == 206) { + + LOGGER.debug("First API Response is 206. Performing iterations..."); + + long requestRangeUpperLimit + = Long.valueOf(requestBuilder.build().header("Range").split("-")[1]) + 1; + long responseRangeUpperLimit = Long.valueOf(responseFlag.getContentRange().split("/")[1]); + + boolean isCredentialLoaded = false; + //config file path for loading credentials + AuthRequest authRequest = new AuthRequest(credentialConfigFilePath, null); + CredentialLoader credentialLoader = new CredentialLoader(authRequest); + AccessToken accessToken = null; + try { + credentialLoader.loadCredentials(); + //Getting access using credentials file as we have to use this for resume capability + accessToken = updateAccessTokenOnTheFly(credentialLoader, feedRequest, requestBuilder); + isCredentialLoaded = true; + } catch (Exception e) { + } + while (requestRangeUpperLimit <= responseRangeUpperLimit) { - LOGGER.debug("API request = {}", request.toString()); + long newUpperLimit = requestRangeUpperLimit + chunkSizeLimit; + String val = "bytes=" + requestRangeUpperLimit + "-" + newUpperLimit; - InvokeResponse responseFlag = invokeIteratively(request, path); + requestBuilder.removeHeader(Constants.RANGE_HEADER); + requestBuilder.addHeader(Constants.RANGE_HEADER, val); - LOGGER.debug("First API Response = {}", responseFlag.toString()); + //this block will not execute if token passed direcrly. Backward compatibile + if (isCredentialLoaded) { + if (checkTokenExpiry(accessToken, credentialLoader, feedRequest, requestBuilder)) { + return new GetFeedResponse(Constants.FAILURE_CODE, Constants.FAILURE, null, null); + } + } - if (responseFlag.getStatusCode() == 200) { - LOGGER.debug("First API Response is 200. All done.."); - return new GetFeedResponse(Constants.SUCCESS_CODE, Constants.SUCCESS, fixFilePath(path,responseFlag), null); + responseFlag = invokeIteratively(requestBuilder.build(), path); + //Showing download progess + LOGGER.debug("First API Response = {}", responseFlag.toString()); - } else if (responseFlag.getStatusCode() == 206) { + if (responseFlag == null) { + return new GetFeedResponse(-1, "Internal error. Please check the logs..", null, null); + } - LOGGER.debug("First API Response is 206. Performing iterations..."); + requestRangeUpperLimit + = Long.valueOf(requestBuilder.build().header(Constants.RANGE_HEADER).split("-")[1]) + 1; + responseRangeUpperLimit = Long.valueOf(responseFlag.getContentRange().split("/")[1]); - long requestRangeUpperLimit = - Long.valueOf(requestBuilder.build().header("Range").split("-")[1]) + 1; - long responseRangeUpperLimit = Long.valueOf(responseFlag.getContentRange().split("/")[1]); + } + return new GetFeedResponse(Constants.SUCCESS_CODE, Constants.SUCCESS, fixFilePath(path, responseFlag), null); + } else { + LOGGER.debug("First API Response is error. Aborting..."); + return new GetFeedResponse(Constants.FAILURE_CODE, Constants.FAILURE, null, null); + } + } - while (requestRangeUpperLimit <= responseRangeUpperLimit) { + /** + * + * @param accessToken + * @param credentialLoader + * @param feedRequest + * @param requestBuilder + * @return + */ + private boolean checkTokenExpiry(AccessToken accessToken, CredentialLoader credentialLoader, FeedRequest feedRequest, Request.Builder requestBuilder) { + try { + Date currentTime = new Date(); + //token refresh on the fly from config file to support resume capability. + if (accessToken.getExpiresOn().before(currentTime)) { + LOGGER.debug("Old Token expiry time = {}", accessToken.getExpiresOn()); + accessToken = updateAccessTokenOnTheFly(credentialLoader, feedRequest, requestBuilder); + LOGGER.info("Got refresh token on the fly and New Token Expiry time = {}", accessToken.getExpiresOn()); + } + } catch (Exception e) { + LOGGER.info("Exception in fetching the new access token " + e.getMessage()); + return true; + } + return false; + } - long newUpperLimit = requestRangeUpperLimit + chunkSizeLimit; - String val = "bytes=" + requestRangeUpperLimit + "-" + newUpperLimit; + /** + *+ * Since date is optional param for getting bootstrap feed, filePath will + * have null value(item_bootstrap-11116-null-EBAY_US.gz) This method helps + * to rename the null value with LastModified api response header + *
+ * + * @param originalFilePath + * @param invokeResponse + * @return + */ + private String fixFilePath(Path originalFilePath, InvokeResponse invokeResponse) { + Path newFilePath = originalFilePath; + if (originalFilePath.toString().contains("null") && !StringUtils.isEmpty(invokeResponse.getLastModified())) { + String newPath = originalFilePath.toString().replace("null", invokeResponse.getLastModified()); + newFilePath = Paths.get(newPath); + try { + Files.move(originalFilePath, newFilePath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + LOGGER.error("Unable to rename the bootstrap item feed file with date field", e); + } + } + return newFilePath.toString(); + } - requestBuilder.removeHeader(Constants.RANGE_HEADER); - requestBuilder.addHeader(Constants.RANGE_HEADER, val); + /** + *+ * Invoked, only if the file size is greater than max chunk size + *
+ * + * @param request The API request + * @param path Path of the downloaded or partially downloading file, where + * contents need to be appended + * @return + */ + private InvokeResponse invokeIteratively(Request request, Path path) { + + InvokeResponse responseFlag = null; + + try (Response response = client.newCall(request).execute()) { + + if (!response.isSuccessful()) { + LOGGER.debug("Error in API response - status = {}, body = {}", response.code(), response + .body().string()); + return new InvokeResponse(null, response.code()); + } - responseFlag = invokeIteratively(requestBuilder.build(), path); + InputStream is = response.body().byteStream(); - if (responseFlag == null) { - return new GetFeedResponse(-1, "Internal error. Please check the logs..", null, null); - } + OutputStream outStream = new FileOutputStream(path.toString(), true); + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = is.read(buffer)) != -1) { + outStream.write(buffer, 0, bytesRead); + } + outStream.flush(); + outStream.close(); + is.close(); + + String lastModifiedHeader = response.header(Constants.LAST_MODIFIED_DATE_HEADER); + String lastModifiedDate = null; + if (!StringUtils.isEmpty(lastModifiedHeader)) { + LocalDate localDate = LocalDate.parse(lastModifiedHeader, DateTimeFormatter.RFC_1123_DATE_TIME); + lastModifiedDate = localDate.format(DateTimeFormatter.BASIC_ISO_DATE); + } - requestRangeUpperLimit = - Long.valueOf(requestBuilder.build().header(Constants.RANGE_HEADER).split("-")[1]) + 1; - responseRangeUpperLimit = Long.valueOf(responseFlag.getContentRange().split("/")[1]); + responseFlag + = new InvokeResponse(response.header(Constants.CONTENT_RANGE_HEADER), response.code(), lastModifiedDate); - } - return new GetFeedResponse(Constants.SUCCESS_CODE, Constants.SUCCESS, fixFilePath(path,responseFlag), null); - } else { - LOGGER.debug("First API Response is error. Aborting..."); - return new GetFeedResponse(Constants.FAILURE_CODE, Constants.FAILURE, null, null); - } - } - - /** - *- * Since date is optional param for getting bootstrap feed, filePath will have null value(item_bootstrap-11116-null-EBAY_US.gz) - * This method helps to rename the null value with LastModified api response header - *
- * @param originalFilePath - * @param invokeResponse - * @return - */ - private String fixFilePath(Path originalFilePath, InvokeResponse invokeResponse) { - Path newFilePath = originalFilePath; - if(originalFilePath.toString().contains("null") && !StringUtils.isEmpty(invokeResponse.getLastModified())){ - String newPath = originalFilePath.toString().replace("null", invokeResponse.getLastModified()); - newFilePath = Paths.get(newPath); - try { - Files.move(originalFilePath, newFilePath, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException e) { - LOGGER.error("Unable to rename the bootstrap item feed file with date field", e); - } - } - return newFilePath.toString(); - } - - /** - *- * Invoked, only if the file size is greater than max chunk size - *
- * - * @param request The API request - * @param path Path of the downloaded or partially downloading file, where contents need to be - * appended - * @return - */ - private InvokeResponse invokeIteratively(Request request, Path path) { - - InvokeResponse responseFlag = null; - - try (Response response = client.newCall(request).execute()) { - - if (!response.isSuccessful()) { - LOGGER.debug("Error in API response - status = {}, body = {}", response.code(), response - .body().string()); - return new InvokeResponse(null, response.code()); - } - - InputStream is = response.body().byteStream(); - - OutputStream outStream = new FileOutputStream(path.toString(), true); - byte[] buffer = new byte[1024]; - int bytesRead; - while ((bytesRead = is.read(buffer)) != -1) { - outStream.write(buffer, 0, bytesRead); - } - outStream.flush(); - outStream.close(); - is.close(); - - String lastModifiedHeader = response.header(Constants.LAST_MODIFIED_DATE_HEADER); - String lastModifiedDate = null; - if(!StringUtils.isEmpty(lastModifiedHeader)){ - LocalDate localDate = LocalDate.parse(lastModifiedHeader, DateTimeFormatter.RFC_1123_DATE_TIME); - lastModifiedDate = localDate.format(DateTimeFormatter.BASIC_ISO_DATE); - } - - responseFlag = - new InvokeResponse(response.header(Constants.CONTENT_RANGE_HEADER), response.code(), lastModifiedDate); - - } catch (Throwable t) { - LOGGER.error("Exception in feed.invokeIteratively()", t); - responseFlag = new InvokeResponse(null, 400); + } catch (Throwable t) { + LOGGER.error("Exception in feed.invokeIteratively()", t); + responseFlag = new InvokeResponse(null, 400); + } + return responseFlag; } - return responseFlag; - } - /* + /* * (non-Javadoc) * * @see com.ebay.feed.api.Feed#processConfigFile(java.lang.String, java.lang.String) - */ - @Override - public List- * Returns a new instance of response, based on the provided inputs - *
- * - * @param statusCode - * @param message - * @param filePath - * @param appliedFilters - * @return - */ - private com.ebay.feed.model.feed.operation.filter.Response createResponse(Integer statusCode, - String message, String filePath, FeedFilterRequest appliedFilters) { - - LOGGER.debug("Returning response status code = {}, message = {}", statusCode, message); - LOGGER.debug("Filtered file = {}, appliedFilters = {}", filePath, - appliedFilters != null ? appliedFilters.toString() : null); - - return new com.ebay.feed.model.feed.operation.filter.Response(statusCode, message, filePath, - appliedFilters); - } + /** + *+ * Returns a new instance of response, based on the provided inputs + *
+ * + * @param statusCode + * @param message + * @param filePath + * @param appliedFilters + * @return + */ + private com.ebay.feed.model.feed.operation.filter.Response createResponse(Integer statusCode, + String message, String filePath, FeedFilterRequest appliedFilters) { + + LOGGER.debug("Returning response status code = {}, message = {}", statusCode, message); + LOGGER.debug("Filtered file = {}, appliedFilters = {}", filePath, + appliedFilters != null ? appliedFilters.toString() : null); + + return new com.ebay.feed.model.feed.operation.filter.Response(statusCode, message, filePath, + appliedFilters); + } + + /** + * Getting accessToken on the fly using credential file to support resume + * capability + * + * @param credentialLoader + * @param environment + * @param feedRequest + * @param requestBuilder + * @return + */ + private AccessToken updateAccessTokenOnTheFly(CredentialLoader credentialLoader, FeedRequest feedRequest, Request.Builder requestBuilder) throws IOException { + AccessToken accessToken = credentialLoader.getOauthResponse(feedRequest.getEnv()).getAccessToken().get(); + //using token which generated from credentials file as we don't know the expiry time of directly passed token + if (!accessToken.getToken().equalsIgnoreCase(feedRequest.getToken())) { + String token = Constants.TOKEN_BEARER_PREFIX + accessToken.getToken(); + requestBuilder.removeHeader(Constants.AUTHORIZATION_HEADER); + requestBuilder.addHeader(Constants.AUTHORIZATION_HEADER, token); + } + return accessToken; + } } diff --git a/src/main/java/com/ebay/feed/api/Taxonomy.java b/src/main/java/com/ebay/feed/api/Taxonomy.java index bb0c721..8222e52 100644 --- a/src/main/java/com/ebay/feed/api/Taxonomy.java +++ b/src/main/java/com/ebay/feed/api/Taxonomy.java @@ -14,7 +14,6 @@ package com.ebay.feed.api; -import java.io.IOException; import com.ebay.feed.model.feed.operation.taxonomy.CategoryResponse; diff --git a/src/main/java/com/ebay/feed/auth/CredentialLoader.java b/src/main/java/com/ebay/feed/auth/CredentialLoader.java index 26ab827..21e5072 100644 --- a/src/main/java/com/ebay/feed/auth/CredentialLoader.java +++ b/src/main/java/com/ebay/feed/auth/CredentialLoader.java @@ -11,14 +11,12 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.auth; import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import com.ebay.feed.example.CombiningFilters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.ebay.api.client.auth.oauth2.CredentialUtil; @@ -30,70 +28,97 @@ /** ** Responsible for loading consumer credentials and generating token. - * - * In order to generate token, the {@link #loadCredentials() loadCredentials} method needs to be - * invoked, which loads the credentials from the provided config file. - * + * + * In order to generate token, the {@link #loadCredentials() loadCredentials} + * method needs to be invoked, which loads the credentials from the provided + * config file. + * * Once the credentials are loaded, the token can be obtained by invoking the * {@link #getOauthResponse() getOauthResponse} - * + * *
- * + * * @author shanganesh * */ public class CredentialLoader { - private static final Logger LOGGER = LoggerFactory.getLogger(CredentialLoader.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CredentialLoader.class); - private AuthRequest authRequest; + private AuthRequest authRequest; - public CredentialLoader(AuthRequest authRequest) { - this.authRequest = authRequest; + public CredentialLoader(AuthRequest authRequest) { + this.authRequest = authRequest; - } + } - /** - *- * Loads the credentials provided in the config file - *
- * - * @throws Exception Thrown if file is not found or credentials are invalid - */ - public void loadCredentials() throws Exception { + /** + *+ * Loads the credentials provided in the config file + *
+ * + * @throws Exception Thrown if file is not found or credentials are invalid + */ + public void loadCredentials() throws Exception { + + if (authRequest == null) { + throw new Exception("null AuthRequest"); + } + + LOGGER.debug("****** Begin loadCredentials with configPath = {}", + authRequest.getConfigFilePath()); + try { + + File f = new File(authRequest.getConfigFilePath()); + FileInputStream fs = new FileInputStream(f); + CredentialUtil.load(fs); + + } catch (Exception e) { + LOGGER.debug("Exception in loadCredentials", e); + throw new Exception("Could not load credentials"); + } + } - if (authRequest == null) - throw new Exception("null AuthRequest"); + /** + *+ * Generates oauth token, based on the loaded credentials + *
+ * + * @return Returns the OAuthResponse which contains the token + * @throws IOException Thrown if file is not found + */ + public OAuthResponse getOauthResponse() throws IOException { + + LOGGER.debug("****** Begin getOauthResponse with configPath = {}", + authRequest.getConfigFilePath()); + OAuth2Api api = new OAuth2Api(); + OAuthResponse res = api.getApplicationToken(Environment.PRODUCTION, authRequest.getScopes()); + return res; - LOGGER.debug("****** Begin loadCredentials with configPath = {}", - authRequest.getConfigFilePath()); - try { + } - File f = new File(authRequest.getConfigFilePath()); - FileInputStream fs = new FileInputStream(f); - CredentialUtil.load(fs); + /** + *+ * Generates oauth token, based on the loaded credentials + *
+ * + * @param environment + * @return Returns the OAuthResponse which contains the token + * @throws IOException Thrown if file is not found + */ + public OAuthResponse getOauthResponse(String environment) throws IOException { + + LOGGER.debug("****** Begin getOauthResponse with configPath = {}", + authRequest.getConfigFilePath()); + OAuth2Api api = new OAuth2Api(); + OAuthResponse res = null; + //checking environment for token + if (environment.equalsIgnoreCase("SANDBOX")) { + res = api.getApplicationToken(Environment.SANDBOX, authRequest.getScopes()); + } else { + res = api.getApplicationToken(Environment.PRODUCTION, authRequest.getScopes()); + } + return res; - } catch (Exception e) { - LOGGER.debug("Exception in loadCredentials", e); - throw new Exception("Could not load credentials"); } - } - - /** - *- * Generates oauth token, based on the loaded credentials - *
- * - * @return Returns the OAuthResponse which contains the token - * @throws IOException Thrown if file is not found - */ - public OAuthResponse getOauthResponse() throws IOException { - - LOGGER.debug("****** Begin getOauthResponse with configPath = {}", - authRequest.getConfigFilePath()); - OAuth2Api api = new OAuth2Api(); - OAuthResponse res = api.getApplicationToken(Environment.PRODUCTION, authRequest.getScopes()); - return res; - - } } diff --git a/src/main/java/com/ebay/feed/cli/FeedCli.java b/src/main/java/com/ebay/feed/cli/FeedCli.java index ba041ed..cc4996d 100644 --- a/src/main/java/com/ebay/feed/cli/FeedCli.java +++ b/src/main/java/com/ebay/feed/cli/FeedCli.java @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.cli; import java.util.ArrayList; @@ -31,8 +30,6 @@ import com.ebay.feed.api.Feed; import com.ebay.feed.api.FeedImpl; import com.ebay.feed.auth.CredentialLoader; -import com.ebay.feed.enums.EnvTypeEnum; -import com.ebay.feed.enums.FeedTypeEnum; import com.ebay.feed.model.feed.download.GetFeedResponse; import com.ebay.feed.model.feed.operation.feed.FeedRequest; import com.ebay.feed.model.feed.operation.feed.FeedRequest.FeedRequestBuilder; @@ -40,283 +37,320 @@ import com.ebay.feed.model.feed.operation.filter.Response; import com.ebay.feed.model.oauth.AuthRequest; - /** ** Wrapper for exposing feed sdk capabilities using CLI *
- * + * * @author shanganesh * */ public class FeedCli { - - private static final Logger LOGGER = LoggerFactory.getLogger(FeedCli.class); - - public static void main(String[] args) throws Exception { - - // ***Definition Stage*** - - // create Options object - Options options = new Options(); - - options.addOption("help", false, "display help"); - - /*** - * options for downloading the file - */ - // date - options.addOption("dt", true, "the date when feed file was generated"); - - // l1 category - options.addOption("c1", true, "the l1 category id of the feed file"); - - // scope - options.addOption("scope", true, - "the feed scope. Available scopes are ALL_ACTIVE or NEWLY_LISTED"); - - // marketplace - options.addOption("mkt", true, - "the marketplace id for which feed is being request. For example - EBAY_US"); - - // token - options.addOption("token", true, "the oauth token for the consumer. Omit the word 'Bearer'"); - - // env - options.addOption("env", true, "Supported environment types are SANDBOX, PRODUCTION"); - /** - * options for filtering the files - */ - - Option filterL2 = - new Option("c2f", true, "list of l2 categories which are used to filter the feed"); - filterL2.hasArgs(); - filterL2.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterL2); - - Option filterL3 = - new Option("c3f", true, "list of l3 categories which are used to filter the feed"); - filterL3.hasArgs(); - filterL3.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterL3); - - Option filterLeaf = - new Option("lf", true, "list of leaf categories which are used to filter the feed"); - filterLeaf.hasArgs(); - filterLeaf.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterLeaf); - - Option filterSeller = - new Option("sellerf", true, "list of seller names which are used to filter the feed"); - filterSeller.hasArgs(); - filterSeller.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterSeller); - - Option filterLocation = - new Option("locf", true, "list of item locations which are used to filter the feed"); - filterLocation.hasArgs(); - filterLocation.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterLocation); - - options.addOption("pricelf", true, "lower limit of the price range for items in the feed"); - - options.addOption("priceuf", true, "upper limit of the price range for items in the feed"); - - - Option filterEpid = - new Option("epidf", true, "list of epids which are used to filter the feed"); - filterEpid.hasArgs(); - filterEpid.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterEpid); - - Option filterInferredEpid = - new Option("iepidf", true, "list of inferred epids which are used to filter the feed"); - filterInferredEpid.hasArgs(); - filterInferredEpid.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterInferredEpid); - - Option filterGtin = - new Option("gtinf", true, "list of gtins which are used to filter the feed"); - filterGtin.hasArgs(); - filterGtin.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterGtin); - - Option filterItem = - new Option("itemf", true, "list of item ids which are used to filter the feed"); - filterItem.hasArgs(); - filterItem.setArgs(Option.UNLIMITED_VALUES); - options.addOption(filterItem); + private static final Logger LOGGER = LoggerFactory.getLogger(FeedCli.class); + + public static void main(String[] args) throws Exception { + + // ***Definition Stage*** + // create Options object + Options options = new Options(); + + options.addOption("help", false, "display help"); + + /** + * * + * options for downloading the file + */ + // date + options.addOption("dt", true, "the date when feed file was generated"); + + // snapshot_date + options.addOption("sdt", true, "the snapshot_date when feed file was generated"); + + // l1 category + options.addOption("c1", true, "the l1 category id of the feed file"); + + // scope + options.addOption("scope", true, + "the feed scope. Available scopes are ALL_ACTIVE or NEWLY_LISTED"); + + // marketplace + options.addOption("mkt", true, + "the marketplace id for which feed is being request. For example - EBAY_US"); + + // token + options.addOption("token", true, "the oauth token for the consumer. Omit the word 'Bearer'"); + + // env + options.addOption("env", true, "Supported environment types are SANDBOX, PRODUCTION"); + + // type + options.addOption("type", true, "Supported feed types are ITEM, ITEM_SNAPSHOT"); + + /** + * options for filtering the files + */ + Option filterL2 + = new Option("c2f", true, "list of l2 categories which are used to filter the feed"); + filterL2.hasArgs(); + filterL2.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterL2); + + Option filterL3 + = new Option("c3f", true, "list of l3 categories which are used to filter the feed"); + filterL3.hasArgs(); + filterL3.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterL3); + + Option filterLeaf + = new Option("lf", true, "list of leaf categories which are used to filter the feed"); + filterLeaf.hasArgs(); + filterLeaf.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterLeaf); + + Option filterSeller + = new Option("sellerf", true, "list of seller names which are used to filter the feed"); + filterSeller.hasArgs(); + filterSeller.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterSeller); + + Option filterLocation + = new Option("locf", true, "list of item locations which are used to filter the feed"); + filterLocation.hasArgs(); + filterLocation.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterLocation); + + options.addOption("pricelf", true, "lower limit of the price range for items in the feed"); + + options.addOption("priceuf", true, "upper limit of the price range for items in the feed"); + + Option filterEpid + = new Option("epidf", true, "list of epids which are used to filter the feed"); + filterEpid.hasArgs(); + filterEpid.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterEpid); + + Option filterInferredEpid + = new Option("iepidf", true, "list of inferred epids which are used to filter the feed"); + filterInferredEpid.hasArgs(); + filterInferredEpid.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterInferredEpid); + + Option filterGtin + = new Option("gtinf", true, "list of gtins which are used to filter the feed"); + filterGtin.hasArgs(); + filterGtin.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterGtin); + + Option filterItem + = new Option("itemf", true, "list of item ids which are used to filter the feed"); + filterItem.hasArgs(); + filterItem.setArgs(Option.UNLIMITED_VALUES); + options.addOption(filterItem); + + /** + * Overrides for file locations + */ + // file location + options.addOption("dl", "downloadlocation", true, + "override for changing the directory where files are downloaded"); + + // oauth related + options.addOption("cl", "credentiallocation", true, + "directory where the credentials file is located"); + + options.addOption("authscopes", "oauthscopes", true, "list of scopes"); + + // ***Parsing Stage*** + // Create a parser + CommandLineParser parser = new DefaultParser(); + + // parse the options passed as command line arguments + CommandLine cmd = parser.parse(options, args); + + // ***Interrogation Stage*** + // hasOptions checks if option is present or not + // populate feed request + FeedRequestBuilder builder = new FeedRequestBuilder(); + + if (cmd.hasOption("dt")) { + builder.date(cmd.getOptionValue("dt")); + } + + if (cmd.hasOption("sdt")) { + builder.snapshotDate(cmd.getOptionValue("sdt")); + } + + if (cmd.hasOption("c1")) { + builder.categoryId(cmd.getOptionValue("c1")); + } + + if (cmd.hasOption("mkt")) { + builder.siteId(cmd.getOptionValue("mkt")); + } + + if (cmd.hasOption("scope")) { + builder.feedScope(cmd.getOptionValue("scope")); + } + + if (cmd.hasOption("token")) { + builder.token("Bearer " + cmd.getOptionValue("token")); + } + + if (cmd.hasOption("env")) { + builder.env(cmd.getOptionValue("env")); + } + + if (cmd.hasOption("type")) { + builder.type(cmd.getOptionValue("type")); + } else { + // Hardcoded item for backword compatibilty + builder.type("item"); + } + FeedRequest feedRequest = builder.build(); + + // populate filter request if available + FeedFilterRequest filterRequest = new FeedFilterRequest(); + + // populate auth request if available + String credentialFile = null; + String optionalDownloadPath = null; + List+ * Begin the process of downloading and filtering feed based on the input + * parameters + *
+ * + * @param feedRequest + * @param filterRequest + * @throws Exception */ + private static void start(FeedRequest feedRequest, FeedFilterRequest filterRequest, + AuthRequest authRequest, String optionalDownloadPath) throws Exception { - // file location - options.addOption("dl", "downloadlocation", true, - "override for changing the directory where files are downloaded"); - - // oauth related - options.addOption("cl", "credentiallocation", true, - "directory where the credentials file is located"); - - options.addOption("authscopes", "oauthscopes", true, "list of scopes"); - - // ***Parsing Stage*** - // Create a parser - CommandLineParser parser = new DefaultParser(); + Feed feed = new FeedImpl(); - // parse the options passed as command line arguments - CommandLine cmd = parser.parse(options, args); + // if token is null, then retrieve from oauth lib and set in request + if (feedRequest.getToken() == null) { + CredentialLoader credentialLoader = new CredentialLoader(authRequest); + credentialLoader.loadCredentials(); + feedRequest.setToken(credentialLoader.getOauthResponse().getAccessToken().get().getToken()); + filterRequest.setToken(credentialLoader.getOauthResponse().getAccessToken().get().getToken()); + } - // ***Interrogation Stage*** - // hasOptions checks if option is present or not + // download feed + GetFeedResponse getFeedResponse = feed.get(feedRequest, optionalDownloadPath); - // populate feed request - FeedRequestBuilder builder = new FeedRequestBuilder(); + if (getFeedResponse.getStatusCode() != 0) { + return; + } - if (cmd.hasOption("dt")) - builder.date(cmd.getOptionValue("dt")); + // unzip + Response unzipResponse = feed.unzip(getFeedResponse.getFilePath()); - if (cmd.hasOption("c1")) - builder.categoryId(cmd.getOptionValue("c1")); + if (unzipResponse.getStatusCode() != 0) { + return; + } - if (cmd.hasOption("mkt")) - builder.siteId(cmd.getOptionValue("mkt")); - - if (cmd.hasOption("scope")) - builder.feedScope(cmd.getOptionValue("scope")); - - if (cmd.hasOption("token")) - builder.token("Bearer " + cmd.getOptionValue("token")); - - if (cmd.hasOption("env")){ - builder.env(cmd.getOptionValue("env")); + // filter + filterRequest.setInputFilePath(unzipResponse.getFilePath()); + Response filterResponse = feed.filter(filterRequest); + LOGGER.info("Filter response = " + filterResponse.toString()); } - - // TODO : Hardcoded item - builder.type(FeedTypeEnum.ITEM); - FeedRequest feedRequest = builder.build(); - - // populate filter request if available - FeedFilterRequest filterRequest = new FeedFilterRequest(); - - // populate auth request if available - String credentialFile = null; - String optionalDownloadPath = null; - List- * Begin the process of downloading and filtering feed based on the input parameters - *
- * - * @param feedRequest - * @param filterRequest - * @throws Exception - */ - private static void start(FeedRequest feedRequest, FeedFilterRequest filterRequest, - AuthRequest authRequest, String optionalDownloadPath) throws Exception { - - Feed feed = new FeedImpl(); - - // if token is null, then retrieve from oauth lib and set in request - if (feedRequest.getToken() == null) { - CredentialLoader credentialLoader = new CredentialLoader(authRequest); - credentialLoader.loadCredentials(); - feedRequest.setToken(credentialLoader.getOauthResponse().getAccessToken().get().getToken()); - filterRequest.setToken(credentialLoader.getOauthResponse().getAccessToken().get().getToken()); - } - - // download feed - GetFeedResponse getFeedResponse = feed.get(feedRequest, optionalDownloadPath); - - if (getFeedResponse.getStatusCode() != 0) - return; - - // unzip - Response unzipResponse = feed.unzip(getFeedResponse.getFilePath()); - - if (unzipResponse.getStatusCode() != 0) - return; - - // filter - filterRequest.setInputFilePath(unzipResponse.getFilePath()); - Response filterResponse = feed.filter(filterRequest); - System.out.println("Filter response = " + filterResponse.toString()); - } } diff --git a/src/main/java/com/ebay/feed/constants/Constants.java b/src/main/java/com/ebay/feed/constants/Constants.java index 126fb28..430d211 100755 --- a/src/main/java/com/ebay/feed/constants/Constants.java +++ b/src/main/java/com/ebay/feed/constants/Constants.java @@ -11,60 +11,69 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.constants; public class Constants { - /** - * Base path for the feed api - */ - public static final String FEED_API_PROD_BASE = "https://api.ebay.com/buy/feed/v1_beta/"; - - public static final String FEED_API_SANDBOX_BASE = "https://api.sandbox.ebay.com/buy/feed/v1_beta/"; - - // max content that can be downloaded in one request, in bytes - public static final Long PROD_CHUNK_SIZE = 104857600L; - public static final Long SANDBOX_CHUNK_SIZE = 10485760L; - - // timeout for http client - public static final Integer TIMEOUT = 30; + /** + * Base path for the feed api + */ + public static final String FEED_API_PROD_BASE = "https://api.ebay.com/buy/feed/v1_beta/"; + + public static final String FEED_API_SANDBOX_BASE = "https://api.sandbox.ebay.com/buy/feed/v1_beta/"; + + // max content that can be downloaded in one request, in bytes + public static final Long PROD_CHUNK_SIZE = 104857600L; + public static final Long SANDBOX_CHUNK_SIZE = 10485760L; + + // timeout for http client + public static final Integer TIMEOUT = 30; + + // api related constants + public static final String AUTHORIZATION_HEADER = "Authorization"; + + // taxonomy + public static final String TAXONOMY_DEFAULT + = "https://api.ebay.com/commerce/taxonomy/v1_beta/get_default_category_tree_id?marketplace_id="; + + public static final String TAXONOMY_CATEGORY_SUBTREE_BASE = "https://api.ebay.com/commerce/taxonomy/v1_beta/category_tree/"; + public static final String TAXONOMY_CATEGORY_SUBTREE_QUERY = "/get_category_subtree?category_id="; + + // status and util related constants + public static final String SUCCESS = "Success"; + public static final Integer SUCCESS_CODE = 0; + public static final String FAILURE = "Failure"; + public static final Integer FAILURE_CODE = -1; + public static final String RANGE_PREFIX = "bytes=0-"; + public static final String RANGE_HEADER = "Range"; + public static final String CONTENT_RANGE_HEADER = "Content-Range"; + public static final String LAST_MODIFIED_DATE_HEADER = "Last-Modified"; + public static final String TOKEN_BEARER_PREFIX = "Bearer "; + public static final String TRUE = "true"; + public static final String FALSE = "false"; + public static final String ITEM_RESOURCE_SCOPE = "item?feed_scope="; + public static final String ITEM_SNAPSHOT_RESOURCE = "item_snapshot?"; + public static final String QUERY_PARAM_SEPARATOR = "&"; + public static final String QUERY_CATEGORY_ID = "category_id="; + public static final String QUERY_DATE = "date="; + public static final String QUERY_SNAPSHOT_DATE = "snapshot_date="; + public static final String QUERY_DATE_FORMAT = "yyyyMMdd"; + public static final String QUERY_SNAPSHOT_DATE_FORMAT = "yyyy-MM-dd'T'HH:00:00.000'Z'"; + public static final String MARKETPLACE_HEADER = "X-EBAY-C-MARKETPLACE-ID"; + public static final String CONTENT_TYPE_HEADER = "Content-type"; + public static final String ACCEPT_HEADER = "Accept"; + + public static final String ALL_ACTIVE = "ALL_ACTIVE"; + public static final String NEWLY_LISTED = "NEWLY_LISTED"; + public static final Integer NEWLY_LISTED_AVAILABILITY_START_DAY = -14; + public static final Integer NEWLY_LISTED_AVAILABILITY_END_DAY = -2; + + public static final Integer ITEM_SNAPSHOT_AVAILABILITY_START_DAY = -7; + public static final Integer ITEM_SNAPSHOT_AVAILABILITY_END_HOUR = -2; - // api related constants - public static final String AUTHORIZATION_HEADER = "Authorization"; - - // taxonomy - public static final String TAXONOMY_DEFAULT = - "https://api.ebay.com/commerce/taxonomy/v1_beta/get_default_category_tree_id?marketplace_id="; - - public static final String TAXONOMY_CATEGORY_SUBTREE_BASE = "https://api.ebay.com/commerce/taxonomy/v1_beta/category_tree/"; - public static final String TAXONOMY_CATEGORY_SUBTREE_QUERY = "/get_category_subtree?category_id="; - - // status and util related constants - public static final String SUCCESS = "Success"; - public static final Integer SUCCESS_CODE = 0; - public static final String FAILURE = "Failure"; - public static final Integer FAILURE_CODE = -1; - public static final String RANGE_PREFIX = "bytes=0-"; - public static final String RANGE_HEADER = "Range"; - public static final String CONTENT_RANGE_HEADER = "Content-Range"; - public static final String LAST_MODIFIED_DATE_HEADER = "Last-Modified"; - public static final String TOKEN_BEARER_PREFIX = "Bearer "; - public static final String TRUE = "true"; - public static final String FALSE = "false"; - public static final String ITEM_RESOURCE_SCOPE = "item?feed_scope="; - public static final String QUERY_PARAM_SEPARATOR = "&"; - public static final String QUERY_CATEGORY_ID = "category_id="; - public static final String QUERY_DATE = "date="; - public static final String MARKETPLACE_HEADER = "X-EBAY-C-MARKETPLACE-ID"; - public static final String CONTENT_TYPE_HEADER = "Content-type"; - public static final String ACCEPT_HEADER = "Accept"; + public static final String BOOTSTRAP = "bootstrap"; + public static final String DAILY = "daily"; + public static final String SEPARATOR = "\\t"; + public static final String ITEM_ID = "ItemId"; - public static final String ALL_ACTIVE = "ALL_ACTIVE"; - - public static final String BOOTSTRAP = "bootstrap"; - public static final String DAILY = "daily"; - public static final String SEPARATOR = "\\t"; - public static final String ITEM_ID = "ItemId"; - } diff --git a/src/main/java/com/ebay/feed/enums/FeedTypeEnum.java b/src/main/java/com/ebay/feed/enums/FeedTypeEnum.java index b37ff03..aac0cba 100644 --- a/src/main/java/com/ebay/feed/enums/FeedTypeEnum.java +++ b/src/main/java/com/ebay/feed/enums/FeedTypeEnum.java @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.enums; /** @@ -23,5 +22,14 @@ */ public enum FeedTypeEnum { - ITEM + ITEM, ITEM_SNAPSHOT; + + public static FeedTypeEnum getFeedEnum(String name) { + for (FeedTypeEnum feedEnum : FeedTypeEnum.values()) { + if (feedEnum.name().equalsIgnoreCase(name)) { + return feedEnum; + } + } + return null; + } } diff --git a/src/main/java/com/ebay/feed/example/CombiningFilters.java b/src/main/java/com/ebay/feed/example/CombiningFilters.java index 385a904..cf2dc66 100644 --- a/src/main/java/com/ebay/feed/example/CombiningFilters.java +++ b/src/main/java/com/ebay/feed/example/CombiningFilters.java @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.example; import java.util.HashSet; @@ -19,19 +18,21 @@ import com.ebay.feed.api.Feed; import com.ebay.feed.api.FeedImpl; import com.ebay.feed.constants.Constants; -import com.ebay.feed.enums.FeedTypeEnum; import com.ebay.feed.model.feed.download.GetFeedResponse; import com.ebay.feed.model.feed.operation.feed.FeedRequest; import com.ebay.feed.model.feed.operation.feed.FeedRequest.FeedRequestBuilder; import com.ebay.feed.model.feed.operation.filter.FeedFilterRequest; import com.ebay.feed.model.feed.operation.filter.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** *
- * Example showing how to download and filter feed files based on a combination of filter criteria
- * The download location is default - current working directory
- * This example downloads the bootstrap feed file for L1 category : 1 (Collectibles) and filters on
- * :-
+ * Example showing how to download and filter feed files based on a combination
+ * of filter criteria The download location is default - current working
+ * directory
+ * This example downloads the bootstrap feed file for L1 category : 1
+ * (Collectibles) and filters on :-
* - leaf categories
* - price
* - item location
@@ -41,93 +42,95 @@
* - Unzip feed file
* - Filter feed file
*
+ * Example showing how to download and filter item_snapshot feed files based on
+ * itemIds. The download location is default - current working directory
+ * This example downloads the item_snapshot feed file for L1 category : 1
+ * (Collectibles) and filters on itemIds.
+ * The filtering is performed on the unzipped file.
+ * So the sequence of events that are followed is :-
+ * - Download feed file
+ * - Unzip feed file
+ * - Filter feed file
+ *
- * Example showing how to download and filter feed files based on environment. The download location
- * is default - current working directory
+ * Example showing how to download and filter feed files based on environment.
+ * The download location is default - current working directory
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on seller user names. The download
- * location is default - current working directory
- * This example downloads the bootstrap feed file for L1 category : 1 (Collectibles) and filters on
- * seller user names.
+ * Example showing how to download and filter feed files based on seller user
+ * names. The download location is default - current working directory
+ * This example downloads the bootstrap feed file for L1 category : 1
+ * (Collectibles) and filters on seller user names.
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Simple example of applying a filter on an already downlaoded file
+ * Simple example of applying a filter on an already downlaoded file
*
* The filtering is performed on the unzipped file.
* - Filter feed file
*
@@ -43,6 +44,9 @@ * */ public class FilterByItemIds { + + private static final Logger LOGGER = LoggerFactory.getLogger(FilterByItemIds.class); + // oauth token - Bearer xxx static String token = @@ -54,31 +58,31 @@ public class FilterByItemIds { private static final String CATEGORY = "1281"; - // TODO : Check if the date is within 14 days, before making the call private static final String DATE = "20180805"; private static final String SCOPE = "ALL_ACTIVE"; private static final String MKT = "EBAY_US"; + private static final String FEEDTYPE = "item"; public static void main(String[] args) { // create request FeedRequest.FeedRequestBuilder builder = new FeedRequestBuilder(); - builder.categoryId(CATEGORY).date(DATE).feedScope(SCOPE).siteId(MKT).token(token) - .type(FeedTypeEnum.ITEM); + builder.categoryId(CATEGORY).date(DATE).feedScope(SCOPE).siteId(MKT).token(token) + .type(FEEDTYPE); // using null for download directory - defaults to current working directory GetFeedResponse getFeedResponse = feed.get(builder.build(), null); // 0 denotes successful response if (getFeedResponse.getStatusCode() != 0) { - System.out.println("Exception in downloading feed. Cannot proceed"); + LOGGER.info("Exception in downloading feed. Cannot proceed"); return; } // unzip Response unzipOpResponse = feed.unzip(getFeedResponse.getFilePath()); if (unzipOpResponse.getStatusCode() != 0) { - System.out.println("Exception in unzipping feed. Cannot proceed"); + LOGGER.info("Exception in unzipping feed. Cannot proceed"); return; } @@ -89,8 +93,8 @@ public static void main(String[] args) { filterRequest.setInputFilePath(unzipOpResponse.getFilePath()); Response response = feed.filter(filterRequest); - System.out.println("Filter status = " + response.getStatusCode()); - System.out.println("Filtered file = " + response.getFilePath()); + LOGGER.info("Filter status = " + response.getStatusCode()); + LOGGER.info("Filtered file = " + response.getFilePath()); } diff --git a/src/main/java/com/ebay/feed/example/FilterByItemLocation.java b/src/main/java/com/ebay/feed/example/FilterByItemLocation.java index 60f65b1..f46879b 100644 --- a/src/main/java/com/ebay/feed/example/FilterByItemLocation.java +++ b/src/main/java/com/ebay/feed/example/FilterByItemLocation.java @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.example; import java.util.HashSet; @@ -19,90 +18,93 @@ import com.ebay.feed.api.Feed; import com.ebay.feed.api.FeedImpl; import com.ebay.feed.constants.Constants; -import com.ebay.feed.enums.FeedTypeEnum; import com.ebay.feed.model.feed.download.GetFeedResponse; import com.ebay.feed.model.feed.operation.feed.FeedRequest; import com.ebay.feed.model.feed.operation.feed.FeedRequest.FeedRequestBuilder; import com.ebay.feed.model.feed.operation.filter.FeedFilterRequest; import com.ebay.feed.model.feed.operation.filter.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** *
- * Example showing how to download and filter feed files based on item location. The download
- * location is default - current working directory
+ * Example showing how to download and filter feed files based on item location.
+ * The download location is default - current working directory
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on leaf category ids. The download
- * location is default - current working directory
+ * Example showing how to download and filter feed files based on leaf category
+ * ids. The download location is default - current working directory
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on L3 category ids. The download
- * location is default - current working directory
- *
- * To filter on L3 category, the mapping between L3 to leaf category ids needs to be loaded. Once
- * the mapping is loaded, the operation translates to actually filtering the feed files on leaf
- * category ids, which belong to the L3 category id.
- *
+ * Example showing how to download and filter feed files based on L3 category
+ * ids. The download location is default - current working directory
+ *
+ * To filter on L3 category, the mapping between L3 to leaf category ids needs
+ * to be loaded. Once the mapping is loaded, the operation translates to
+ * actually filtering the feed files on leaf category ids, which belong to the
+ * L3 category id.
+ *
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on item price. The download location
- * is default - current working directory
+ * Example showing how to download and filter feed files based on item price.
+ * The download location is default - current working directory
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on seller user names. The download
- * location is default - current working directory
- * This example downloads the bootstrap feed file for L1 category : 1 (Collectibles) and filters on
- * seller user names.
+ * Example showing how to download and filter feed files based on seller user
+ * names. The download location is default - current working directory
+ * This example downloads the bootstrap feed file for L1 category : 1
+ * (Collectibles) and filters on seller user names.
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
- * Example showing how to download and filter feed files based on item location. The download
- * location is default - current working directory
+ * Example showing how to download and filter feed files based on item location.
+ * The download location is default - current working directory
* The filtering is performed on the unzipped file.
* So the sequence of events that are followed is :-
* - Download feed file
* - Unzip feed file
* - Filter feed file
*
+ * Example showing how to download and filter feed files based on item location.
+ * The download location is default - current working directory
+ * The filtering is performed on the unzipped file.
+ * So the sequence of events that are followed is :-
+ * - Download feed file
+ * - Unzip feed file
+ * - Filter feed file
+ *
+ * Example showing how to download and filter feed files based on seller user
+ * names. The download location is default - current working directory
+ * This example downloads the bootstrap feed file for L1 category : 1
+ * (Collectibles) and filters on seller user names.
+ * The filtering is performed on the unzipped file.
+ * So the sequence of events that are followed is :-
+ * - Download feed file
+ * - Unzip feed file
+ * - Filter feed file
+ *
* Utils class for performing utility functions for feed impl *
- * + * * @author shanganesh * */ public class FeedUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(FeedUtil.class); - - - /** - *- * Generate URL based on the feed request values - *
- * - * @param feedRequest Container for capturing parameters related to downloading and unzipping feed files - * @return string Creates a url for invoking feed api, based on the inputs - */ - protected String getFinalUrl(FeedRequest feedRequest) { - - String finalUrl = null; - FeedTypeEnum type = feedRequest.getType(); - - String baseUrl = getBaseUrl(feedRequest); - StringBuilder bdr = new StringBuilder(baseUrl); - switch (type) { - - case ITEM: - - bdr.append(Constants.ITEM_RESOURCE_SCOPE); - bdr.append(feedRequest.getFeedScope()); - bdr.append(Constants.QUERY_PARAM_SEPARATOR + Constants.QUERY_CATEGORY_ID); - bdr.append(feedRequest.getCategoryId()); - bdr.append(Constants.QUERY_PARAM_SEPARATOR + Constants.QUERY_DATE); - bdr.append(feedRequest.getDate()); - finalUrl = bdr.toString(); - break; - - default: - break; + private static final Logger LOGGER = LoggerFactory.getLogger(FeedUtil.class); + + /** + *+ * Generate URL based on the feed request values + *
+ * + * @param feedRequest Container for capturing parameters related to + * downloading and unzipping feed files + * @return string Creates a url for invoking feed api, based on the inputs + */ + protected String getFinalUrl(FeedRequest feedRequest) { + + String finalUrl = null; +// FeedTypeEnum type = feedRequest.getType(); + FeedTypeEnum type = FeedTypeEnum.getFeedEnum(feedRequest.getType()); + + String baseUrl = getBaseUrl(feedRequest); + StringBuilder bdr = new StringBuilder(baseUrl); + switch (type) { + + case ITEM: + + bdr.append(Constants.ITEM_RESOURCE_SCOPE); + bdr.append(feedRequest.getFeedScope()); + bdr.append(Constants.QUERY_PARAM_SEPARATOR + Constants.QUERY_CATEGORY_ID); + bdr.append(feedRequest.getCategoryId()); +// if (feedRequest.getFeedScope().equalsIgnoreCase(Constants.NEWLY_LISTED)) { + bdr.append(Constants.QUERY_PARAM_SEPARATOR + Constants.QUERY_DATE); + bdr.append(feedRequest.getDate()); +// } + finalUrl = bdr.toString(); + break; + + case ITEM_SNAPSHOT: + + bdr.append(Constants.ITEM_SNAPSHOT_RESOURCE); + bdr.append(Constants.QUERY_CATEGORY_ID); + bdr.append(feedRequest.getCategoryId()); + bdr.append(Constants.QUERY_PARAM_SEPARATOR + Constants.QUERY_SNAPSHOT_DATE); + bdr.append(feedRequest.getSnapshotDate()); + finalUrl = bdr.toString(); + break; + + default: + break; + } + + LOGGER.debug("Final URL for API = {}", finalUrl); + return finalUrl; } - LOGGER.debug("Final URL for API = {}", finalUrl); - return finalUrl; - } + private String getBaseUrl(FeedRequest feedRequest) { + EnvTypeEnum env = EnvTypeEnum.getEnvEnum(feedRequest.getEnv()); + String baseUrl = Constants.FEED_API_PROD_BASE; + if (env != null && env == EnvTypeEnum.SANDBOX) { + baseUrl = Constants.FEED_API_SANDBOX_BASE; + } + return baseUrl; + } - private String getBaseUrl(FeedRequest feedRequest) { - EnvTypeEnum env = EnvTypeEnum.getEnvEnum(feedRequest.getEnv()); - String baseUrl = Constants.FEED_API_PROD_BASE; - if(env != null && env == EnvTypeEnum.SANDBOX){ - baseUrl = Constants.FEED_API_SANDBOX_BASE; + public Long getChunkSizeLimit(FeedRequest feedRequest) { + EnvTypeEnum env = EnvTypeEnum.getEnvEnum(feedRequest.getEnv()); + Long chunkSize = Constants.PROD_CHUNK_SIZE; + if (env != null && env == EnvTypeEnum.SANDBOX) { + chunkSize = Constants.SANDBOX_CHUNK_SIZE; + } + return chunkSize; } - return baseUrl; - } - - public Long getChunkSizeLimit(FeedRequest feedRequest) { - EnvTypeEnum env = EnvTypeEnum.getEnvEnum(feedRequest.getEnv()); - Long chunkSize = Constants.PROD_CHUNK_SIZE; - if(env != null && env == EnvTypeEnum.SANDBOX){ - chunkSize = Constants.SANDBOX_CHUNK_SIZE; - } - return chunkSize; - } -/** - *- * Generate request instance for the feed API call - *
- * - * @param feedRequest Container for capturing parameters related to downloading and unzipping feed files - * @param requestBuilder Returns a builder - * @return RequestBuilder Returns a builder - */ - public Request.Builder generateRequest(FeedRequest feedRequest, Request.Builder requestBuilder) { - // static headers - requestBuilder.addHeader(Constants.MARKETPLACE_HEADER, feedRequest.getMarketplaceId()); - requestBuilder.addHeader(Constants.CONTENT_TYPE_HEADER, "application/json"); - requestBuilder.addHeader(Constants.ACCEPT_HEADER, "application/json"); - - // url - requestBuilder.url(getFinalUrl(feedRequest)); - - // token - requestBuilder.addHeader(Constants.AUTHORIZATION_HEADER, feedRequest.getToken()); - return requestBuilder; - } - - /** - *Generates a file name based on the input parameters
- * - * @param feedRequest Container for capturing parameters related to downloading and unzipping feed files - * @return string Generates a filename based on the feed type, scope, category, marketplace and date - */ - public String generateFileName(FeedRequest feedRequest) { - - String fileType = - Constants.ALL_ACTIVE.equalsIgnoreCase(feedRequest.getFeedScope()) ? Constants.BOOTSTRAP - : Constants.DAILY; - - switch (feedRequest.getType()) { - case ITEM: - return "item_" + fileType + "-" + feedRequest.getCategoryId() + "-" + feedRequest.getDate() - + "-" + feedRequest.getMarketplaceId() + ".gz"; - default: - return null; + /** + *+ * Generate request instance for the feed API call + *
+ * + * @param feedRequest Container for capturing parameters related to + * downloading and unzipping feed files + * @param requestBuilder Returns a builder + * @return RequestBuilder Returns a builder + */ + public Request.Builder generateRequest(FeedRequest feedRequest, Request.Builder requestBuilder) { + // static headers + requestBuilder.addHeader(Constants.MARKETPLACE_HEADER, feedRequest.getMarketplaceId()); + requestBuilder.addHeader(Constants.CONTENT_TYPE_HEADER, "application/json"); + requestBuilder.addHeader(Constants.ACCEPT_HEADER, "application/json"); + + // url + requestBuilder.url(getFinalUrl(feedRequest)); + + // token + requestBuilder.addHeader(Constants.AUTHORIZATION_HEADER, feedRequest.getToken()); + return requestBuilder; + } + + /** + *+ * Generates a file name based on the input parameters
+ * + * @param feedRequest Container for capturing parameters related to + * downloading and unzipping feed files + * @return string Generates a filename based on the feed type, scope, + * category, marketplace and date + */ + public String generateFileName(FeedRequest feedRequest) { + + String fileType + = Constants.ALL_ACTIVE.equalsIgnoreCase(feedRequest.getFeedScope()) ? Constants.BOOTSTRAP + : Constants.DAILY; + + FeedTypeEnum type = FeedTypeEnum.getFeedEnum(feedRequest.getType()); + + // switch (feedRequest.getType()) { + switch (type) { + case ITEM: + return "item_" + fileType + "-" + feedRequest.getCategoryId() + "-" + feedRequest.getDate() + + "-" + feedRequest.getMarketplaceId() + ".gz"; + case ITEM_SNAPSHOT: + return "item_snapshot-" + feedRequest.getCategoryId() + "-" + feedRequest.getSnapshotDate().replaceAll(":", "") + + "-" + feedRequest.getMarketplaceId() + ".gz"; + default: + return null; + } } - } - /** - *- * Unzips a file and creates a new file and returns the path Returns null in case of errors - *
- * @param path Path to the downloaded compressed feed file - * @return string Path to the unzipped file - * @throws Exception Exception - */ - public String unzip(Path path) throws Exception { + /** + *+ * Unzips a file and creates a new file and returns the path Returns null in + * case of errors + *
+ * + * @param path Path to the downloaded compressed feed file + * @return string Path to the unzipped file + * @throws Exception Exception + */ + public String unzip(Path path) throws Exception { - if (path == null) - return null; + if (path == null) { + return null; + } - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[1024]; - try { + try { - LOGGER.debug("Starting unzipping operation for = {}", path.toString()); - String newFilePath = (path.toString().substring(0, path.toString().length() - 3)); + LOGGER.debug("Starting unzipping operation for = {}", path.toString()); + String newFilePath = (path.toString().substring(0, path.toString().length() - 3)); - GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(path.toFile())); + GZIPInputStream gzis = new GZIPInputStream(new FileInputStream(path.toFile())); - FileOutputStream out = new FileOutputStream(newFilePath); + FileOutputStream out = new FileOutputStream(newFilePath); - int len; - while ((len = gzis.read(buffer)) > 0) { - out.write(buffer, 0, len); - } + int len; + while ((len = gzis.read(buffer)) > 0) { + out.write(buffer, 0, len); + } - gzis.close(); - out.close(); + gzis.close(); + out.close(); - LOGGER.debug("Completed unzipping operation. Unzipped file = {}", newFilePath); - return newFilePath; + LOGGER.debug("Completed unzipping operation. Unzipped file = {}", newFilePath); + return newFilePath; - } catch (IOException ex) { - LOGGER.debug("Exception in unzip()", ex); - ex.printStackTrace(); + } catch (IOException ex) { + LOGGER.debug("Exception in unzip()", ex); + ex.printStackTrace(); + } + return null; } - return null; - } } diff --git a/src/main/java/com/ebay/feed/util/FilterUtil.java b/src/main/java/com/ebay/feed/util/FilterUtil.java index 2670da5..4f96192 100644 --- a/src/main/java/com/ebay/feed/util/FilterUtil.java +++ b/src/main/java/com/ebay/feed/util/FilterUtil.java @@ -11,7 +11,6 @@ * or implied. See the License for the specific language governing permissions and limitations under * the License. */ - package com.ebay.feed.util; import java.io.BufferedReader; @@ -28,6 +27,7 @@ import com.ebay.feed.api.Taxonomy; import com.ebay.feed.api.TaxonomyImpl; import com.ebay.feed.constants.Constants; +import com.ebay.feed.enums.FeedTypeEnum; import com.ebay.feed.model.feed.operation.filter.FeedFilterRequest; import com.ebay.feed.model.feed.operation.taxonomy.CategoryResponse; @@ -35,60 +35,59 @@ ** Utility class to perform filtering on feed files *
- * + * * @author shanganesh * */ public class FilterUtil { - private static final Logger LOGGER = LoggerFactory.getLogger(FilterUtil.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FilterUtil.class); - /** - *- * Apply the filters to the contents specified in the baseFilePath - *
- * - * @param baseFilePath Path to the unzipped feed file - * @param filterRequest Container for capturing the filter parameters - * @return string The path of the filtered file - * @throws Exception exception - */ - public String filter(Path baseFilePath, FeedFilterRequest filterRequest) throws Exception { + /** + *+ * Apply the filters to the contents specified in the baseFilePath + *
+ * + * @param baseFilePath Path to the unzipped feed file + * @param filterRequest Container for capturing the filter parameters + * @return string The path of the filtered file + * @throws Exception exception + */ + public String filter(Path baseFilePath, FeedFilterRequest filterRequest) throws Exception { - LOGGER.debug("******* Begin filtering on file = {} with parameters = {}", baseFilePath, - filterRequest); + LOGGER.debug("******* Begin filtering on file = {} with parameters = {}", baseFilePath, + filterRequest); - String filteredFile = getFilteredFileName(baseFilePath, filterRequest); + String filteredFile = getFilteredFileName(baseFilePath, filterRequest); - try (BufferedReader r = new BufferedReader(new FileReader(baseFilePath.toFile())); - BufferedWriter w = new BufferedWriter(new FileWriter(filteredFile))) { + try (BufferedReader r = new BufferedReader(new FileReader(baseFilePath.toFile())); + BufferedWriter w = new BufferedWriter(new FileWriter(filteredFile))) { - String line = null; + String line = null; - while ((line = r.readLine()) != null) { - String[] tsv = line.split(Constants.SEPARATOR); + while ((line = r.readLine()) != null) { + String[] tsv = line.split(Constants.SEPARATOR); - // only for header - if (evaluateHeader(tsv)) { - w.write(line + System.lineSeparator()); - } + // only for header + if (evaluateHeader(tsv)) { + w.write(line + System.lineSeparator()); + } - // actual listings - if (evaluate(tsv, filterRequest)) { - w.write(line + System.lineSeparator()); - } - } + // actual listings + if (evaluate(tsv, filterRequest)) { + w.write(line + System.lineSeparator()); + } + } - } catch (Exception e) { - LOGGER.error("Error in FilterUtils.filter()", e); - throw e; + } catch (Exception e) { + LOGGER.error("Error in FilterUtils.filter()", e); + throw e; + } + return filteredFile; } - return filteredFile; - } - - /** - * + /** + *- * Evaluate if the line is a headerline - *
- * - * @param line One record from the feed file - * @return boolean Checks if the line is a header line - */ - private boolean evaluateHeader(String[] line) { - return line[0].contains(Constants.ITEM_ID); - } - - /** - *+ * + * @param line One record from the feed file + * @param filterRequest Container for capturing the filter parameters + * @return boolean Indicates whether any filters apply to this record + */ + protected boolean evaluate(String[] line, FeedFilterRequest filterRequest) { + FeedTypeEnum type = FeedTypeEnum.getFeedEnum(filterRequest.getType()); + switch (type) { + case ITEM: + return evaluateField(line, filterRequest.getLeafCategoryIds(), 4) + && evaluateField(line, filterRequest.getSellerNames(), 6) + && evaluateField(line, filterRequest.getItemLocationCountries(), 21) + && evaluateItemPrice(line, filterRequest) + && evaluateField(line, filterRequest.getEpids(), 12) + && evaluateField(line, filterRequest.getInferredEpids(), 40) + && evaluateField(line, filterRequest.getGtins(), 9) + && evaluateField(line, filterRequest.getItemIds(), 0); + + case ITEM_SNAPSHOT: + return evaluateField(line, filterRequest.getLeafCategoryIds(), 5) + && evaluateField(line, filterRequest.getSellerNames(), 7) + && evaluateField(line, filterRequest.getItemLocationCountries(), 22) + && evaluateItemPrice(line, filterRequest) + && evaluateField(line, filterRequest.getEpids(), 13) + && evaluateField(line, filterRequest.getInferredEpids(), 58) + && evaluateField(line, filterRequest.getGtins(), 10) + && evaluateField(line, filterRequest.getItemIds(), 0) + && checkItemAvailability(line, 1); + default: + return false; + } + } + + /** + *
+ * Evaluate if the line is a headerline + *
+ * + * @param line One record from the feed file + * @return boolean Checks if the line is a header line + */ + private boolean evaluateHeader(String[] line) { + return line[0].contains(Constants.ITEM_ID); + } + + /** + ** Evaluate if the item price based on the inputs - within a range - higher than a limit - lower * than given limit - *
- * - * @param line One record from the feed file - * @param filterRequest Container for capturing the filter parameters - * @return boolean Checks if the price filter applies to this item - */ - private boolean evaluateItemPrice(String[] line, FeedFilterRequest filterRequest) { - - boolean isValid = true; - - if (filterRequest == null - || (filterRequest.getPriceLowerLimit() == null && filterRequest.getPriceUpperLimit() == null)) - return isValid; - - Double price = null; - int colNo = 15; - - if (!isColumnValid(colNo, line.length)) - return isValid; - - try { - price = Double.valueOf(line[colNo]); - } catch (Exception e) { - return false; - } + * + * + * @param line One record from the feed file + * @param filterRequest Container for capturing the filter parameters + * @return boolean Checks if the price filter applies to this item + */ + private boolean evaluateItemPrice(String[] line, FeedFilterRequest filterRequest) { + + boolean isValid = true; + + if (filterRequest == null + || (filterRequest.getPriceLowerLimit() == null && filterRequest.getPriceUpperLimit() == null)) + return isValid; + + Double price = null; + int colNo = 15; + if (filterRequest.getType().equalsIgnoreCase(FeedTypeEnum.ITEM_SNAPSHOT.name())) { + colNo = 16; + } + + if (!isColumnValid(colNo, line.length)) + return isValid; - // between range - if (filterRequest.getPriceLowerLimit() != null && filterRequest.getPriceUpperLimit() != null) { + try { + price = Double.valueOf(line[colNo]); + } catch (Exception e) { + return false; + } + + // between range + if (filterRequest.getPriceLowerLimit() != null && filterRequest.getPriceUpperLimit() != null) { - if (price < filterRequest.getPriceLowerLimit() || price > filterRequest.getPriceUpperLimit()) { - isValid = false; + if (price < filterRequest.getPriceLowerLimit() || price > filterRequest.getPriceUpperLimit()) { + isValid = false; - } - } else if (filterRequest.getPriceLowerLimit() != null) { + } + } else if (filterRequest.getPriceLowerLimit() != null) { - if (price < filterRequest.getPriceLowerLimit()) - isValid = false; + if (price < filterRequest.getPriceLowerLimit()) + isValid = false; - } else if (filterRequest.getPriceUpperLimit() != null) { + } else if (filterRequest.getPriceUpperLimit() != null) { - if (price > filterRequest.getPriceUpperLimit()) - isValid = false; + if (price > filterRequest.getPriceUpperLimit()) + isValid = false; + } + return isValid; } - return isValid; - } - - - /** - *- * Evaluates the presence of field at 'columnNo', is present in the given set. - * - * This method is generic for handling filtering on 'set of string' filters. - *
- * - * @param line One record from the feed file - * @param filterSet Set of filters - * @param column Column to filter on - * @return - */ - private boolean evaluateField(String[] line, Set+ * Evaluates the presence of field at 'columnNo', is present in the given set. + * + * This method is generic for handling filtering on 'set of string' filters. + *
+ * + * @param line One record from the feed file + * @param filterSet Set of filters + * @param column Column to filter on + * @return + */ + private boolean evaluateField(String[] line, Set+ * Evaluates the presence of field at 'columnNo', is present in the given set. + * + * This method is for skipping UNAVAILABLE items + *
+ * + * @param line One record from the feed file + * @param column Column to filter on + * @return + */ + private boolean checkItemAvailability(String[] line, Integer columnNo) { + boolean isValid = true; + + // guard against parsing issues + if (!isColumnValid(columnNo, line.length)) + return isValid; + + String field = line[columnNo]; + + if(columnNo == 1 && field.equalsIgnoreCase("UNAVAILABLE")) + isValid = false; + + return isValid; + + } + + /** + *+ * Generate filtered file name based on base file path and timestamp + *
+ * + * @param baseFilePath Path of the unzipped file + * @param filterRequest Container for capturing the filter parameters + * @return + */ + private String getFilteredFileName(Path baseFilePath, FeedFilterRequest filterRequest) { + Date d = new Date(); + String filteredFile = baseFilePath.toString() + "-filtered-" + d.getTime(); + return filteredFile; } - return isValid; - - } - - /** - *- * Generate filtered file name based on base file path and timestamp - *
- * - * @param baseFilePath Path of the unzipped file - * @param filterRequest Container for capturing the filter parameters - * @return - */ - private String getFilteredFileName(Path baseFilePath, FeedFilterRequest filterRequest) { - Date d = new Date(); - String filteredFile = baseFilePath.toString() + "-filtered-" + d.getTime(); - return filteredFile; - } - - /** - *- * Calculate leaf categories if level two or level three category filters have been specified - * - *
- * - * @param filterRequest Container for capturing the filter parameters - * @return Set of leaf categories - */ - public Set+ * Calculate leaf categories if level two or level three category filters have been specified + * + *
+ * + * @param filterRequest Container for capturing the filter parameters + * @return Set of leaf categories + */ + public Set- * Get map of leaf categories - *
- * - * @param map Map of level one to children categories - * @param catSet Set of leaf categories - * @return Set of leaf categories - */ - private Set+ * Get map of leaf categories + *
+ * + * @param map Map of level one to children categories + * @param catSet Set of leaf categories + * @return Set of leaf categories + */ + private Set* Class responsible for validating feed and filter requests *
- * + * * @author shanganesh * */ public class FeedValidator { - /** - * Check path validity - * - * @param filePath Checks if the file exists on the specified path - * @return boolean Indicates whether the file was found on the specified path - */ - public boolean isValidPath(String filePath) { - - if (filePath == null || filePath.isEmpty()) - return false; - - Path path = Paths.get(filePath); - - return Files.exists(path); - } - - /** - *- * Check feed filter request - *
- * - * @param filterRequest Container for capturing parameters related to filtering the feed files - * @return boolean Indicates whether all the mandatory parameters are present - */ - public boolean isValidFilterRequest(FeedFilterRequest filterRequest) { - - if (filterRequest == null) - return false; + private static final Logger LOGGER = LoggerFactory.getLogger(FeedValidator.class); + SimpleDateFormat sdf; + Calendar calendar; - if (filterRequest.getInputFilePath() == null || filterRequest.getInputFilePath().isEmpty()) - return false; + /** + * Check path validity + * + * @param filePath Checks if the file exists on the specified path + * @return boolean Indicates whether the file was found on the specified + * path + */ + public boolean isValidPath(String filePath) { - Path path = Paths.get(filterRequest.getInputFilePath()); + if (filePath == null || filePath.isEmpty()) { + return false; + } - if (!Files.exists(path)) - return false; - - if (filterRequest.isEmpty()) - return false; - - if (!isValidLevelNRequest(filterRequest)) - return false; - - return true; - } + Path path = Paths.get(filePath); + return Files.exists(path); + } - /** - * Check that the mandatory parameters are present, in case filtering on level 'n' is requested - * - * @return - */ - private boolean isValidLevelNRequest(FeedFilterRequest filterRequest) { + /** + *+ * Check feed filter request + *
+ * + * @param filterRequest Container for capturing parameters related to + * filtering the feed files + * @return boolean Indicates whether all the mandatory parameters are + * present + */ + public boolean isValidFilterRequest(FeedFilterRequest filterRequest) { + + if (filterRequest == null) { + return false; + } + + if (filterRequest.getInputFilePath() == null || filterRequest.getInputFilePath().isEmpty()) { + return false; + } + + Path path = Paths.get(filterRequest.getInputFilePath()); + + if (!Files.exists(path)) { + return false; + } + + if (filterRequest.isEmpty()) { + return false; + } + + if (!isValidLevelNRequest(filterRequest)) { + return false; + } + + return true; + } - boolean isValid = true; - // check if level 'n' categories are present. - if ((filterRequest.getLevelTwoCategories() != null && !filterRequest.getLevelTwoCategories() - .isEmpty()) - || (filterRequest.getLevelThreeCategories() != null && !filterRequest - .getLevelThreeCategories().isEmpty())) { + /** + * Check that the mandatory parameters are present, in case filtering on + * level 'n' is requested + * + * @return + */ + private boolean isValidLevelNRequest(FeedFilterRequest filterRequest) { + + boolean isValid = true; + // check if level 'n' categories are present. + if ((filterRequest.getLevelTwoCategories() != null && !filterRequest.getLevelTwoCategories() + .isEmpty()) + || (filterRequest.getLevelThreeCategories() != null && !filterRequest + .getLevelThreeCategories().isEmpty())) { + + if (filterRequest.getToken() == null || filterRequest.getMarketplace() == null + || filterRequest.getLevelOneCategory() == null) { + isValid = false; + } + + } + return isValid; + } - if (filterRequest.getToken() == null || filterRequest.getMarketplace() == null - || filterRequest.getLevelOneCategory() == null) - isValid = false; + /** + * Check that the date format is valid and date is within valid range for + * selected feed type + * + * Feed Type Item date format is yyyyMMdd and range would be within previous + * 3-14 days + * + * Feed Type ItemSnapshot date format is yyyy-MM-dd'T'HH:00:00.000'Z' and + * range would be within previous 7 days + * + * @param feedRequest + * @return + */ + public boolean isValidDateFormatAndRange(FeedRequest feedRequest) { + boolean isValid = true; + FeedTypeEnum type = FeedTypeEnum.getFeedEnum(feedRequest.getType()); + + switch (type) { + case ITEM: + if (Constants.NEWLY_LISTED.equalsIgnoreCase(feedRequest.getFeedScope())) { + try { + sdf = new SimpleDateFormat(Constants.QUERY_DATE_FORMAT); + Date feedItemRequestDate = sdf.parse(feedRequest.getDate()); + LOGGER.debug("feedItemRequestDate : {}", feedItemRequestDate); + + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT")); + LOGGER.debug("Current Time in GMT Time Zone : {}", calendar.getTime()); +// Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DAY_OF_MONTH, Constants.NEWLY_LISTED_AVAILABILITY_START_DAY); + Date feedItemAvailabilityStartDate = sdf.parse(sdf.format(calendar.getTime())); + LOGGER.debug("feedItemAvailabilityStartDate : {}", feedItemAvailabilityStartDate); + + calendar = Calendar.getInstance(); + calendar.add(Calendar.DAY_OF_MONTH, Constants.NEWLY_LISTED_AVAILABILITY_END_DAY); + Date feedItemAvailabilityEndDate = sdf.parse(sdf.format(calendar.getTime())); + LOGGER.debug("feedItemAvailabilityEndDate : {}", feedItemAvailabilityEndDate); + + if (!isWithinRange(feedItemRequestDate, feedItemAvailabilityStartDate, feedItemAvailabilityEndDate)) { + isValid = false; + } + // good format and within valid range + } catch (ParseException ex) { + isValid = false; + LOGGER.error("Item Date format is not valid. Cannot get feed. Aborting...", ex); + } + } + + break; + + case ITEM_SNAPSHOT: + try { + sdf = new SimpleDateFormat(Constants.QUERY_SNAPSHOT_DATE_FORMAT); + Date feedItemSnapshotRequestDate = sdf.parse(feedRequest.getSnapshotDate()); + LOGGER.debug("feedItemSnapshotRequestDate : {}", feedItemSnapshotRequestDate); + + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT")); + LOGGER.debug("Current Time in GMT Time Zone : {}", calendar.getTime()); + //Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DAY_OF_MONTH, Constants.ITEM_SNAPSHOT_AVAILABILITY_START_DAY); + Date feedItemSnapshotAvailabilitStartDate = sdf.parse(sdf.format(calendar.getTime())); + LOGGER.debug("feedItemSnapshotAvailabilitStartDate : {}", feedItemSnapshotAvailabilitStartDate); + + calendar = Calendar.getInstance(); + calendar.add(Calendar.HOUR_OF_DAY, Constants.ITEM_SNAPSHOT_AVAILABILITY_END_HOUR); + Date feedItemSnapshotAvailabilitEndDate = sdf.parse(sdf.format(calendar.getTime())); + LOGGER.debug("feedItemSnapshotAvailabilitEndDate : {}", feedItemSnapshotAvailabilitEndDate); + + if (!isWithinRange(feedItemSnapshotRequestDate, feedItemSnapshotAvailabilitStartDate, feedItemSnapshotAvailabilitEndDate)) { + isValid = false; + } + // good format and within valid range + } catch (ParseException ex) { + isValid = false; + LOGGER.error("Item Snapshot Date format is not valid. Cannot get feed. Aborting...", ex); + } + break; + + default: + break; + } + return isValid; } - return isValid; - } - + /** + * Feed Type Item date range would be within previous 3-14 days + * + * Feed Type ItemSnapshot date range would be within previous 7 days + * + * @param queryRequestDate + * @param startDate + * @param endDate + * @return + */ + private boolean isWithinRange(Date queryRequestDate, Date startDate, Date endDate) { + return !(queryRequestDate.before(startDate) || queryRequestDate.after(endDate)); + } } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index ac7aff9..f509dd0 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -1,48 +1,48 @@ +~ /** +~ * Copyright (c) 2018 eBay Inc. +~ * +~ * Licensed under the Apache License, Version 2.0 (the "License"); +~ * you may not use this file except in compliance with the License. +~ * You may obtain a copy of the License at +~ * +~ * http://www.apache.org/licenses/LICENSE-2.0 +~ * +~ * Unless required by applicable law or agreed to in writing, software +~ * distributed under the License is distributed on an "AS IS" BASIS, +~ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +~ * See the License for the specific language governing permissions and +~ * limitations under the License. +~ **/ +-->