/**
 * Performs static initialization for Objectify to register types and do other setup.
 *
 * <p>This method is non-idempotent,so it should only be called exactly once,which is achieved
 * by calling it from this class's static initializer block.
 */
private static void initOfyOnce() {
  // Set an ObjectifyFactory that uses our extended ObjectifyImpl.
  // The "false" argument means that we are not using the v5-style Objectify embedded entities.
  com.googlecode.objectify.ObjectifyService.setFactory(new ObjectifyFactory(false) {
    @Override
    public Objectify begin() {
      return new SessionKeyExposingObjectify(this);
    }

    @Override
    protected AsyncDatastoreService createrawAsyncDatastoreService(DatastoreServiceConfig cfg) {
      // In the unit test environment,wrap the Datastore service in a proxy that can be used to
      // examine the number of requests sent to Datastore.
      AsyncDatastoreService service = super.createrawAsyncDatastoreService(cfg);
      return RegistryEnvironment.get().equals(RegistryEnvironment.UNITTEST)
          ? new RequestCapturingAsyncDatastoreService(service)
          : service;
    }});

  // Translators must be registered before any entities can be registered.
  registerTranslators();
  registerEntityClasses(EntityClasses.ALL_CLASSES);
}

@Test
public void testMiscops() throws Exception {
    AsyncDatastoreService service = DatastoreServiceFactory.getAsyncDatastoreService();

    DatastoreAttributes attributes = waitOnFuture(service.getDatastoreAttributes());
    Assert.assertNotNull(attributes);
    Assert.assertNotNull(attributes.getDatastoreType());

    Map<Index,Index.IndexState> indexes = waitOnFuture(service.getIndexes());
    Assert.assertNotNull(indexes);

    Transaction tx = waitOnFuture(service.beginTransaction());
    try {
        String txId = tx.getId();
        Assert.assertNotNull(txId);
        Assert.assertEquals(txId,tx.getId());

        String appId = tx.getApp();
        Assert.assertNotNull(appId);
        Assert.assertEquals(appId,tx.getApp());
    } finally {
        tx.rollback();
    }
}

protected <T> T inTx(Action<T> action) throws Exception {
    AsyncDatastoreService ads = DatastoreServiceFactory.getAsyncDatastoreService();
    Transaction tx = ads.beginTransaction().get();
    boolean ok = false;
    try {
        T result = action.run(ads);
        ok = true;
        return result;
    } finally {
        if (ok)
            tx.commitAsync();
        else
            tx.rollbackAsync();

        sync(); // wait for tx to finish
    }
}

@Test
public void testCRUD() throws Exception {
    AsyncDatastoreService service = DatastoreServiceFactory.getAsyncDatastoreService();

    Entity e1 = new Entity("ASYNC");
    e1.setProperty("foo","bar");

    Future<Key> k1 = service.put(e1);
    Assert.assertNotNull(k1);

    Key key = k1.get();
    Assert.assertNotNull(key);
    Future<Entity> fe1 = service.get(key);
    Assert.assertNotNull(fe1);
    Assert.assertEquals(e1,fe1.get());

    Future<Void> fd1 = service.delete(key);
    Assert.assertNotNull(fd1);
    fd1.get();

    assertStoreDoesNotContain(key);
}

@Test
public void testCommitTx() throws Exception {
    AsyncDatastoreService service = DatastoreServiceFactory.getAsyncDatastoreService();
    Transaction tx = waitOnFuture(service.beginTransaction(Transactionoptions.Builder.withDefaults()));
    Key key;
    try {
        Future<Key> fKey = service.put(tx,new Entity("AsyncTx"));
        key = waitOnFuture(fKey);
        waitOnFuture(tx.commitAsync());
    } catch (Exception e) {
        waitOnFuture(tx.rollbackAsync());
        throw e;
    }

    if (key != null) {
        Assert.assertNotNull(getSingleEntity(service,key));
    }
}

@Test
public void testRollbackTx() throws Exception {
    AsyncDatastoreService service = DatastoreServiceFactory.getAsyncDatastoreService();
    Transaction tx = waitOnFuture(service.beginTransaction(Transactionoptions.Builder.withDefaults()));
    Key key = null;
    try {
        Future<Key> fKey = service.put(tx,new Entity("AsyncTx"));
        key = waitOnFuture(fKey);
    } finally {
        waitOnFuture(tx.rollbackAsync());
    }

    if (key != null) {
        Assert.assertNull(getSingleEntity(service,key));
    }
}

@Override
public void putBlobsAsync(Map<I_BlobKey,I_Blob> values) throws BlobException
{
    List<Entity> entities = createEntitiesForPut(values);

    AsyncDatastoreService datastore = DatastoreServiceFactory.getAsyncDatastoreService();
    Transaction currentTransaction = m_blobTxnMngr.getCurrentTransaction(datastore);

    if( currentTransaction != null )
    {
        throw new BlobException("Can't do async multi-put within a blob transaction.");
    }

    try
    {
        datastore.put(entities);
    }
    catch(Exception e)
    {
        throw new BlobException("Error occurred while putting blob batch.",e);
    }
}

@Override
public void deleteBlobAsync(I_BlobKey keySource,Class<? extends I_Blob> blobType) throws BlobException
{
    AsyncDatastoreService datastore = DatastoreServiceFactory.getAsyncDatastoreService();

    Transaction currentTransaction = m_blobTxnMngr.getCurrentTransaction(datastore);

    if( currentTransaction != null )
    {
        throw new BlobException("Can't delete within transaction.");
    }

    I_Blob blobTemplate = m_templateMngr.getTemplate(blobType);

    Key keyObject = KeyFactory.createKey(blobTemplate.getKind(),keySource.createBlobKey(blobTemplate));

    try
    {
        datastore.delete(keyObject);
    }
    catch(Exception e)
    {
        throw new BlobException("UnkNown error occurred while deleting blob (async).",e);
    }
}

@Override
public void deleteBlobsAsync(Map<I_BlobKey,Class<? extends I_Blob>> values) throws BlobException
{
    AsyncDatastoreService datastore = DatastoreServiceFactory.getAsyncDatastoreService();

    Transaction currentTransaction = m_blobTxnMngr.getCurrentTransaction(datastore);

    if( currentTransaction != null )
    {
        throw new BlobException("Can't delete within transaction.");
    }

    List<Key> keys = createKeysForBatchDelete(values);

    try
    {
        datastore.delete(keys);
    }
    catch(Exception e)
    {
        throw new BlobException("UnkNown error occurred while deleting blobs (async).",e);
    }
}

@Test
public void testBeginTx() throws Exception {
    final AsyncDatastoreService service = DatastoreServiceFactory.getAsyncDatastoreService();

    Transaction tx = waitOnFuture(service.beginTransaction(Transactionoptions.Builder.withXG(true)));
    Key key,key2;
    try {
        key = waitOnFuture(service.put(tx,new Entity("AsyncTx")));
        key2 = waitOnFuture(service.put(tx,new Entity("AsyncTx")));
        tx.commit();
    } catch (Exception e) {
        tx.rollback();
        throw e;
    }

    if (key != null && key2 != null) {
        tx = waitOnFuture(service.beginTransaction(Transactionoptions.Builder.withXG(true)));
        try {
            try {
                try {
                    Assert.assertNotNull(waitOnFuture(service.get(tx,key)));
                    Assert.assertNotNull(waitOnFuture(service.get(tx,Collections.singleton(key2))));
                } finally {
                    service.delete(tx,key2);
                }
            } finally {
                service.delete(tx,Collections.singleton(key));
            }
        } finally {
            tx.rollback();
        }
    }
}

@Test
public void testFutureCancel() {
    clearData();
    AsyncDatastoreService asyncService = DatastoreServiceFactory.getAsyncDatastoreService();
    Entity newRec = new Entity(ASYNC_ENTITY);
    newRec.setProperty("count",-1);
    newRec.setProperty("timestamp",new Date());
    Future<Key> future = asyncService.put(newRec);
    future.cancel(true);

    // The actual call may already succeeded,so just verify that cancel has been called.
    assertTrue(future.isCancelled());
}

@Override
public void putBlobAsync(I_BlobKey keySource,I_Blob blob) throws BlobException
{
    AsyncDatastoreService datastore = DatastoreServiceFactory.getAsyncDatastoreService();

    Transaction currentTransaction = m_blobTxnMngr.getCurrentTransaction(datastore);

    if( currentTransaction != null )
    {
        throw new BlobException("Blob manager does not support transactions for async puts.");
    }

    Entity entity = createEntityForPut(keySource,blob);

    try
    {
        datastore.put(entity);
    }
    catch(ConcurrentModificationException concurrencyException)
    {
        throw concurrencyException;
    }
    catch(Exception e)
    {
        throw new BlobException("Some error occured while putting blob.",e);
    }
}

RequestCapturingAsyncDatastoreService(AsyncDatastoreService delegate) {
  this.delegate = delegate;
}

@Deprecated
public static AsyncDatastoreService getInstance() {
  return DatastoreServiceFactory.getAsyncDatastoreService();
}

protected Entity getSingleEntity(AsyncDatastoreService ds,Key key) {
    Map<Key,Entity> map = waitOnFuture(ds.get(Collections.singleton(key)));
    return (map.isEmpty() ? null : map.values().iterator().next());
}

T run(AsyncDatastoreService ads) throws Exception;

public Entity buildEntityFromPersistentObject(final P persistentObject,final DAO<P> dao) {
    if (persistentObject == null) {
        return null;
    }

    final Entity anEntity = new Entity(dao.getEntityName(),persistentObject.getId());

    final byte[] binaryData = objectHolderSerializer.serialize(persistentObject.getDataObject(),true);

    if (binaryData.length > 1000000) {
        throw new RuntimeException("BinaryData length for object [" + persistentObject + "] is bigger than permitted: " + binaryData.length);
    }

    anEntity.setUnindexedProperty(DATA_CONTAINER_PROPERTY,new Blob(binaryData));

    for (final PropertyMeta propertyMeta : persistentObject.getPropertiesMetadata().values()) {
        if (propertyMeta.isIndexable()) {
            anEntity.setProperty(propertyMeta.getPropertyName(),propertyMeta.get());
        }
    }
    return anEntity;
}

public static Resource load(final String id,final boolean exceptionOnNotFound) {
    Resource result = (Resource) CacheStrategy.APPLICATION_CACHE.get().get(ENTITY_NAME + id);

    if (result == null) {
        final Entity entity = new retryingHandler().tryDSGet(DAO.buildKey(ENTITY_NAME,id));

        if (entity != null) {
            result = new Resource();
            result.id = id;
            result.type = (String) entity.getProperty("type");

            final Blob binaryData = (Blob) entity.getProperty("bdata");
            result.rawValue = binaryData == null ? null : binaryData.getBytes();
        }
    }

    if (result == null && exceptionOnNotFound) {
        throw new RuntimeException("Could not find resource with key [" + id + "]");
    }
    return result;
}

public void save(final boolean saveAsync) {
    validateSize();

    final Entity anEntity = new Entity(ENTITY_NAME,id);

    if (type != null) {
        anEntity.setProperty("type",type);
    }
    if (rawValue != null) {
        anEntity.setUnindexedProperty("bdata",new Blob(rawValue));
    }

    final retryingHandler retryingHandler = new retryingHandler();

    if (saveAsync) {
        retryingHandler.tryDSPutAsync(anEntity);
    } else {
        retryingHandler.tryDSPut(anEntity);
    }
    CacheStrategy.APPLICATION_CACHE.get().put(ENTITY_NAME + id,this);
}

public  List<Event> GetEventsForAggregate(UUID aggregateId)
{
    logger.log(Level.INFO,ComplexLogMessages.eventStoreReadMessage("AggregateId",aggregateId.toString()));

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    Key aggregateKey = KeyFactory.createKey("WishListEvents",aggregateId.toString());

    Query query = new Query("WishListEvent",aggregateKey);
    List<Entity> storedEvents = datastore.prepare(query).asList(FetchOptions.Builder.withDefaults());

    ArrayList<Event> eventData = new ArrayList<Event>();
    for(Entity e : storedEvents)
    {
        eventData.add(PropertyConverter.convertBlobToObject(Event.class,(Blob)e.getProperty("EventData")));
    }

    if(eventData.isEmpty())
    {
        logger.log(Level.WARNING,ComplexLogMessages.exceptionMessage("AggregateNotFoundException","Aggregate was not found."));
        throw new AggregateNotFoundException("Aggregate was not found.");
    }

    return eventData;
}

@Override
public List<WishListDto> GetWishLists() 
{
    logger.log(Level.INFO,ComplexLogMessages.queryFacadeMessage("GetWishLists"));

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    Key aggregateKey = KeyFactory.createKey("WishListsView","WishListsView");

    Query query = new Query("WishListDto",aggregateKey);
       List<Entity> storedEntities = datastore.prepare(query).asList(FetchOptions.Builder.withDefaults());

       List<WishListDto> wishLists = new ArrayList<WishListDto>();

       for(Entity e : storedEntities)
       {
        wishLists.add(PropertyConverter.convertBlobToObject(WishListDto.class,(Blob)e.getProperty("WishListDto")));
       }
    return wishLists;
}

public void handle(NameOfWishListChangedEvent event)
{
    logger.log(Level.INFO,ComplexLogMessages.eventHandlerMessage(event.getClass().getSimpleName(),"Id",event.Id.toString(),"NewName",event.NewName,"Version","" + event.Version));

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    Key aggregateKey = KeyFactory.createKey("WishListsView",aggregateKey);
       List<Entity> storedEntities = datastore.prepare(query).asList(FetchOptions.Builder.withDefaults());

       for(Entity e : storedEntities)
       {
        if(UUID.fromString((String)e.getProperty("Id")).equals(event.Id))
        {
            WishListDto wishListDto = PropertyConverter.convertBlobToObject(WishListDto.class,(Blob)e.getProperty("WishListDto"));
            wishListDto.setName(event.NewName);
            wishListDto.setVersion(event.Version);

            e.setProperty("WishListDto",PropertyConverter.convertObjectToBlob(wishListDto));
            datastore.put(e);
            break;
        }
       }
}

public void handle(WishListClearedEvent event)
{
    logger.log(Level.INFO,(Blob)e.getProperty("WishListDto"));
            wishListDto.getItems().clear();
            wishListDto.setVersion(event.Version);

            e.setProperty("WishListDto",PropertyConverter.convertObjectToBlob(wishListDto));
            datastore.put(e);
            break;
        }
       }
}

public void handle(ItemAddedToWishListEvent event)
{
    logger.log(Level.INFO,"ItemId",event.ItemId.toString(),"ItemName",event.ItemName,"ItemPrice","" + event.ItemPrice,(Blob)e.getProperty("WishListDto"));
            wishListDto.getItems().add(new Triple<UUID,String,Double>(event.ItemId,event.ItemPrice));
            wishListDto.setVersion(event.Version);

            e.setProperty("WishListDto",PropertyConverter.convertObjectToBlob(wishListDto));
            datastore.put(e);
            break;
        }
       }
}

public static <T>Blob convertObjectToBlob(T o)
  {

    try 
    {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        ObjectOutput out = new ObjectOutputStream(bos);
    out.writeObject(o);
    return new Blob(bos.toByteArray());
} 
    catch (IOException e) 
    {
    e.printstacktrace();
    return null;
}
  }

@SuppressWarnings("unchecked")
public static <T>T convertBlobToObject(Class<T> clazz,Blob b)
   {

    try 
    {
        ByteArrayInputStream bos = new ByteArrayInputStream(b.getBytes());
        ObjectInputStream is = new ObjectInputStream(bos);
        return (T) is.readobject();
    } 
    catch (IOException | ClassNotFoundException e) 
    {
        e.printstacktrace();
        return null;
    }
   }

@Test
public void testSize() {
    String kind = kindName + "-size";
    int recordSize = (1000 * 1000);  // Max. 1000000.
    byte[] filledRec = new byte[recordSize];
    Arrays.fill(filledRec,(byte) 0x41);
    Blob bigBlob = new Blob(filledRec);
    assertEquals(recordSize,bigBlob.getBytes().length);
    Entity eBlob = new Entity(kind,rootKey);
    eBlob.setProperty("blobProp",bigBlob);
    service.put(eBlob);

    recordSize = 500 ;  // Max. 500.
    filledRec = new byte[recordSize];
    Arrays.fill(filledRec,(byte) 0x41);
    ShortBlob shortBlob = new ShortBlob(filledRec);
    assertEquals(recordSize,shortBlob.getBytes().length);
    Entity eShortBlob = new Entity(kind,rootKey);
    eShortBlob.setProperty("byteStrProp",shortBlob);
    service.put(eShortBlob);

    service.delete(eBlob.getKey(),eShortBlob.getKey());
}

private Entity createEntityForPut(I_BlobKey keySource,I_Blob blob) throws BlobException
{
    Key keyObject = KeyFactory.createKey(blob.getKind(),keySource.createBlobKey(blob));
    Entity entity = new Entity(keyObject);
    byte[] blobBytes = U_Blob.convertToBytes(blob);
    Blob blobData = new Blob(blobBytes);
    entity.setUnindexedProperty(S_Blob.DATA_FIELD_NAME,blobData);

    Map<String,Object> queryableProperties = blob.getQueryableProperties();

    if( queryableProperties != null )
    {
        for( String key : queryableProperties.keySet() )
        {
            entity.setProperty(key,queryableProperties.get(key));
        }
    }

    return entity;
}

private static SimpleModule getWriteBlobAsBase64Module() {
  JsonSerializer<Blob> dateSerializer = new JsonSerializer<Blob>() {
    @Override
    public void serialize(Blob value,JsonGenerator jgen,SerializerProvider provider)
        throws IOException {
      byte[] bytes = value.getBytes();
      jgen.writeBinary(bytes,bytes.length);
    }
  };
  SimpleModule writeBlobAsBase64Module = new SimpleModule("writeBlobAsBase64Module",new Version(1,null,null));
  writeBlobAsBase64Module.addSerializer(Blob.class,dateSerializer);
  return writeBlobAsBase64Module;
}

@Test
public void testReadBlobParameter() throws Exception {
  Method method =
      TestEndpoint.class.getDeclaredMethod("doblob",Blob.class);
  Object[] params = readParameters("{\"blob\":\"AQIDBA==\"}",method);

  assertEquals(1,params.length);
  assertthat(((Blob) params[0]).getBytes()).isEqualTo(new byte[]{1,2,3,4});
}

@SuppressWarnings("unused")
public void testBlobAsBase64() throws Exception {
  Object value = new Object() {
    public Blob getBlob() {
      return new Blob(new byte[]{1,4});
    }
  };
  ObjectNode output = ObjectMapperUtil.createStandardobjectMapper()
      .readValue(writetoResponse(value),ObjectNode.class);
  assertEquals("AQIDBA==",output.path("blob").asText());
}

@Override
public Entity toEntity() {
  Entity entity = toProtoEntity();
  entity.setUnindexedProperty(SHARD_ID_PROPERTY,shardId);
  entity.setUnindexedProperty(VALUE_PROPERTY,new Blob(value));
  return entity;
}

public ExceptionRecord(Entity entity) {
  super(entity);
  Blob serializedExceptionBlob = (Blob) entity.getProperty(EXCEPTION_PROPERTY);
  byte[] serializedException = serializedExceptionBlob.getBytes();
  try {
    exception = (Throwable) SerializationUtils.deserialize(serializedException);
  } catch (IOException e) {
    throw new RuntimeException("Failed to deserialize exception for " + getKey(),e);
  }
}

@Override
public Entity toEntity() {
  try {
    Entity entity = toProtoEntity();
    byte[] serializedException = SerializationUtils.serialize(exception);
    entity.setUnindexedProperty(EXCEPTION_PROPERTY,new Blob(serializedException));
    return entity;
  } catch (IOException e) {
    throw new RuntimeException("Failed to serialize exception for " + getKey(),e);
  }
}

@Override
public Object serializeValue(PipelineModelObject model,Object value) throws IOException {
  byte[] bytes = SerializationUtils.serialize(value);
  if (bytes.length < MAX_BLOB_BYTE_SIZE) {
    return new Blob(bytes);
  }
  int shardId = 0;
  int offset = 0;
  final List<Entity> shardedValues = new ArrayList<>(bytes.length / MAX_BLOB_BYTE_SIZE + 1);
  while (offset < bytes.length) {
    int limit = offset + MAX_BLOB_BYTE_SIZE;
    byte[] chunk = Arrays.copyOfRange(bytes,offset,Math.min(limit,bytes.length));
    offset = limit;
    shardedValues.add(new ShardedValue(model,shardId++,chunk).toEntity());
  }
  return tryFiveTimes(new Operation<List<Key>>("serializeValue") {
    @Override
    public List<Key> call() {
      Transaction tx = dataStore.beginTransaction();
      List<Key> keys;
      try {
        keys = dataStore.put(tx,shardedValues);
        tx.commit();
      } finally {
        if (tx.isActive()) {
          tx.rollback();
        }
      }
      return keys;
    }
  });
}

static SessionData createSessionFromEntity(Entity entity) {
  SessionData data = new SessionData();
  data.setExpirationTime((Long) entity.getProperty(EXPIRES_PROP));

  Blob valueBlob = (Blob) entity.getProperty(VALUES_PROP);
  @SuppressWarnings("unchecked")
  Map<String,Object> valueMap = (Map<String,Object>) deserialize(valueBlob.getBytes());
  data.setValueMap(valueMap);
  return data;
}

/**
 * Return an {@link Entity} for the given key and data in the empty
 * namespace.
 */
static Entity createEntityForSession(String key,SessionData data) {
  String originalNamespace = NamespaceManager.get();
  try {
    NamespaceManager.set("");
    Entity entity = new Entity(SESSION_ENTITY_TYPE,key);
    entity.setProperty(EXPIRES_PROP,data.getExpirationTime());
    entity.setProperty(VALUES_PROP,new Blob(serialize(data.getValueMap())));
    return entity;
  } finally {
    NamespaceManager.set(originalNamespace);
  }
}

public P buildPersistentObjectFromEntity(final Entity entity,final DAO<P> dao) {
    P result = null;

    if (entity != null) {
        result = dao.buildPersistentObjectInstance();

        result.setId(entity.getKey().getName());

        final Blob binaryData = (Blob) entity.getProperty(DATA_CONTAINER_PROPERTY);

        if (binaryData != null) {
            objectHolderSerializer.deserialize(binaryData.getBytes(),result.getDataObject(),true);
        }

        for (final PropertyMeta propertyMeta : result.getPropertiesMetadata().values()) {
            if (propertyMeta.isIndexable()) {
                final Serializable propertyValue = (Serializable) entity.getProperty(propertyMeta.getPropertyName());

                if (propertyValue != null && propertyValue.getClass().getName().equals(Long.class.getName()) && propertyMeta.getClass().getName().equals(IntegerProperty.class.getName())) {
                    final Long longValue = (Long) propertyValue;

                    if (longValue > Integer.MAX_VALUE || longValue < Integer.MIN_VALUE) {
                        throw new RuntimeException("Trying to set long value to IntegerProperty. Value was [" + longValue + "],property was [" + propertyMeta.getPropertyName() + "]");
                    }

                    propertyMeta.set(longValue.intValue());
                } else {
                    propertyMeta.set(propertyValue);
                }
            }
        }
    }
    return result;
}

public static Map<UUID,List<Event>> GetEvents()
{
    Map<UUID,List<Event>> aggregates = new LinkedHashMap<UUID,List<Event>>();
    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();

    Query query = new Query("WishListEvent").addSort("Date",Query.sortDirection.DESCENDING);
    List<Entity> storedEvents = datastore.prepare(query).asList(FetchOptions.Builder.withDefaults());

    for(Entity e : storedEvents)
    {
        Event event = PropertyConverter.convertBlobToObject(Event.class,(Blob)e.getProperty("EventData"));         

        if(aggregates.containsKey(event.Id))
        {
            List<Event> eventData = aggregates.get(event.Id);
            eventData.add(0,event);
        }
        else
        {
            List<Event> savedEventData = new ArrayList<Event>();
            savedEventData.add(event);
            aggregates.put(event.Id,savedEventData);
        }
    }

    return aggregates;
}

public void handle(WishListCreatedEvent event)
{
    logger.log(Level.INFO,"Name",event.Name,"" + event.Version));

    Key aggregateKey = KeyFactory.createKey("WishListsView","WishListsView");

    Blob wishListDtoBlob = PropertyConverter.convertObjectToBlob(new WishListDto(event.Id,new ArrayList<Triple<UUID,Double>>(),0));

    Entity entity = new Entity("WishListDto",aggregateKey);
    entity.setProperty("Id",event.Id.toString());
    entity.setProperty("WishListDto",wishListDtoBlob);

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    datastore.put(entity);
}

public void handle(ItemRemovedFromWishListEvent event)
{
    logger.log(Level.INFO,(Blob)e.getProperty("WishListDto"));
            wishListDto.setVersion(event.Version);
            for(int i = 0; i < wishListDto.getItems().size(); i++ )
            {
                Triple<UUID,Double> p = wishListDto.getItems().get(i);
                if(p.getElement1().equals(event.ItemId))
                {
                    wishListDto.getItems().remove(i);
                    break;
                }
            }

            e.setProperty("WishListDto",PropertyConverter.convertObjectToBlob(wishListDto));
            datastore.put(e);
            break;
        }
       }
}

@Test
public void testBlobType() {
    String propertyName = "blobProp";
    List<Entity> elist = doQuery(kindName,propertyName,false);
    Blob blob = (Blob) elist.get(0).getProperty(propertyName);
    Blob sameDat = (Blob) elist.get(0).getProperty(propertyName);
    Blob diffDat = (Blob) elist.get(1).getProperty(propertyName);
    assertTrue(blob.equals(sameDat));
    assertFalse(blob.equals(diffDat));
    byte[] blobData = blob.getBytes();
    assertTrue(Arrays.equals("blobImage".getBytes(),blobData) ||
               Arrays.equals("blobText".getBytes(),blobData) ||
               Arrays.equals("blobData".getBytes(),blobData));
    assertEquals(blob.hashCode(),blob.hashCode());
}

@Test
public void testUnindexedProperties() throws Exception {
    Entity entity = new Entity(UNINDEXED_ENTITY);
    entity.setUnindexedProperty("unindexedString","unindexedString");
    entity.setUnindexedProperty("unindexedList",new ArrayList<String>(Arrays.asList("listElement1","listElement2","listElement3")));
    entity.setUnindexedProperty("unindexedText",new Text("unindexedText"));
    entity.setUnindexedProperty("unindexedBlob",new Blob("unindexedBlob".getBytes()));
    entity.setProperty("text",new Text("text"));
    entity.setProperty("blob",new Blob("blob".getBytes()));

    Key key = service.put(entity);
    sync(3000);  // Not using ancestor queries,so pause before doing queries below.
    Entity entity2 = service.get(key);

    assertTrue(isUnindexed(getRawProperty(entity2,"unindexedString")));
    assertTrue(isUnindexed(getRawProperty(entity2,"unindexedList")));
    assertTrue(isUnindexed(getRawProperty(entity2,"unindexedText")));
    assertTrue(isUnindexed(getRawProperty(entity2,"unindexedBlob")));
    assertTrue(isUnindexed(getRawProperty(entity2,"text")));
    assertTrue(isUnindexed(getRawProperty(entity2,"blob")));

    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedString",EQUAL,"unindexedString"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedList","listElement1"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("unindexedText","unindexedText"))));
    assertNull(getResult(new Query(UNINDEXED_ENTITY).setFilter(new FilterPredicate("text","text"))));

    service.delete(key);
}

static void readBytes(I_Blob blob,Entity entity) throws BlobException
{
    Blob blobData = (Blob) entity.getProperty(S_Blob.DATA_FIELD_NAME);

    byte[] blobBytes = blobData.getBytes();

    U_Blob.readBytes(blob,blobBytes);
}

@Override
protected ThumbnailData parse(Entity e) {
  return new ThumbnailData(new BlobKey(e.getKey().getName()),DatastoreUtil.getExistingProperty(e,THUMBNAIL_BYTES_PROPERTY,Blob.class).getBytes());
}

@Override
protected void populateEntity(ThumbnailData in,Entity out) {
  DatastoreUtil.setNonNullUnindexedProperty(
      out,new Blob(in.thumbnailBytes));
}

@Override
public Blob deserialize(JsonParser jsonParser,DeserializationContext context)
    throws IOException {
  return new Blob(jsonParser.getBinaryValue());
}

public void updateRowFromGae(CommonFieldsBase row,com.google.appengine.api.datastore.Entity gaeEntity) {
  row.setopaquePersistenceData(gaeEntity);
  row.setFromDatabase(true);
  for (datafield d : row.getFieldList()) {
    Object o = gaeEntity.getProperty(d.getName());
    if (o != null) {
      switch (d.getDataType()) {
      case BINARY:
        Blob bin = (Blob) o;
        byte[] array = bin.getBytes();
        if (array != null && array.length != 0) {
          row.setBlobField(d,array);
        }
        break;
      case LONG_STRING:
        Text txt = (Text) o;
        row.setStringField(d,txt.getValue());
        break;
      case BOOLEAN:
        Boolean bool = (Boolean) o;
        row.setBooleanField(d,bool);
        break;
      case DATETIME:
        Date date = (Date) o;
        row.setDateField(d,date);
        break;
      case DECIMAL:
        WrappedBigDecimal bd = WrappedBigDecimal.fromDouble((Double) o);
        row.setNumericField(d,bd);
        break;
      case INTEGER:
        Long l = (Long) o;
        row.setLongField(d,l);
        break;
      case STRING:
      case URI:
        String s = (String) o;
        o = gaeEntity.getProperty("__" + d.getName());
        if (o != null) {
          Text t = (Text) o;
          row.setStringField(d,t.getValue());
        } else {
          row.setStringField(d,s);
        }
        break;
      default:
        throw new IllegalStateException("Unrecognized datatype");
      }
    }
  }
}

private com.google.appengine.api.datastore.Entity prepareGaeFromrow(CommonFieldsBase entity,User user) {

  if (entity.isFromDatabase()) {
    entity.setDateField(entity.lastUpdateDate,new Date());
    entity.setStringField(entity.lastUpdateUriUser,user.getUriUser());
  } else {
    // we need to create the backing object...
    //
    com.google.appengine.api.datastore.Entity gaeEntity;
    // because we sometimes access the nested records without first
    // accessing the top level record,it seems we can't Leverage
    // the Google BigTable parent-key feature for colocation unless
    // we want to take a hit on the getEntity call and turn that
    // into a query.
    gaeEntity = new com.google.appengine.api.datastore.Entity(constructGaeKind(entity),entity.getUri());

    entity.setopaquePersistenceData(gaeEntity);
  }

  // get the google backing object...
  com.google.appengine.api.datastore.Entity e;
  if (entity.getopaquePersistenceData() == null) {
    throw new IllegalStateException("Entity should have opaque persistence data!");
  } else {
    e = (com.google.appengine.api.datastore.Entity) entity.getopaquePersistenceData();
    for (datafield d : entity.getFieldList()) {
      if (entity.isNull(d)) {
        e.removeProperty(d.getName());
      } else
        switch (d.getDataType()) {
        case BINARY:
          byte[] array = entity.getBlobField(d);
          if (array == null || array.length == 0) {
            e.removeProperty(d.getName());
          } else {
            Blob bin = new Blob(array);
            e.setProperty(d.getName(),bin);
          }
          break;
        case LONG_STRING:
          Text txt = new Text(entity.getStringField(d));
          e.setProperty(d.getName(),txt);
          break;
        case BOOLEAN:
          e.setProperty(d.getName(),entity.getBooleanField(d));
          break;
        case DATETIME:
          e.setProperty(d.getName(),entity.getDateField(d));
          break;
        case DECIMAL: 
          WrappedBigDecimal bd = entity.getNumericField(d);
          if ( bd == null ) {
            e.setProperty(d.getName(),null);
          } else {
            e.setProperty(d.getName(),bd.doubleValue());
          }
          break;
        case INTEGER:
          e.setProperty(d.getName(),entity.getLongField(d));
          break;
        case STRING:
        case URI:
          String s = entity.getStringField(d);
          if (s.length() > GAE_MAX_STRING_LEN.intValue()) {
            Text t = new Text(s);
            e.setProperty("__" + d.getName(),t);
            e.setProperty(d.getName(),s.substring(0,GAE_MAX_STRING_LEN.intValue()));
          } else {
            e.removeProperty("__" + d.getName());
            e.setProperty(d.getName(),s);
          }
          break;
        default:
          throw new IllegalStateException("Unrecognized datatype");
        }
    }
  }
  return e;
}

@Override
protected void service(HttpServletRequest req,HttpServletResponse resp)
    throws servletexception,IOException {
    Calendar cal = Calendar.getInstance();
    cal.set(Calendar.HOUR,0);
    cal.set(Calendar.MINUTE,0);
    cal.set(Calendar.SECOND,0);
    cal.set(Calendar.MILLISECOND,0);

    cal.add(Calendar.DATE,-10);

    Downloaders downloaders = new Downloaders();
    DataService data = new DataService();

    for (int i = 0; i < 10; i++) {
        List<Puzzle> puzzles = downloaders.getPuzzles(cal.getTime());
        System.out.println("Got " + puzzles.size() + " puzzles for " +
            cal.getTime());

        for (Puzzle puz : puzzles) {
            PuzzleListing listing = data.findPuzzleListingBySourceAndDate(puz.getSource(),cal.getTime());

            if (listing != null) {
                System.out.println("Puzzle from " + puz.getSource() +
                    " already in database.");
            } else {
                System.out.println("Persisting from " + puz.getSource() +
                    ".");
                listing = new PuzzleListing();
                listing.setDate(cal.getTime());
                listing.setSource(puz.getSource());
                listing.setTitle(puz.getTitle());

                ByteArrayOutputStream puzData = new ByteArrayOutputStream();
                ByteArrayOutputStream Meta = new ByteArrayOutputStream();
                IO.save(puz,new DataOutputStream(puzData),new DataOutputStream(Meta));
                listing.setPuzzleSerial(new Blob(puzData.toByteArray()));
                listing.setMetaSerial(new Blob(Meta.toByteArray()));
                data.store(listing);
            }
        }

        cal.add(Calendar.DATE,1);
    }

    data.close();
    PuzzleServlet.CACHE.put("puzzle-list",null);
}

public void setContent( Blob content ) {
 this.content = content;
}

public Blob getContent() {
 return content;
}

public Blob getimage() {
    return image;
}

public void setimage(Blob image) {
    this.image = image;
}

private void customTask( final HttpServletRequest request,final PersistenceManager pm ) throws IOException {
    LOGGER.fine( "Key: " + request.getParameter( "key" ) + ",file type: " + request.getParameter( "fileType" ) );

    final FileType fileType = FileType.fromString( request.getParameter( "fileType" ) );
    if ( fileType == null ) {
        LOGGER.severe( "Invalid File type!" );
        return;
    }

    final Key key = KeyFactory.stringToKey( request.getParameter( "key" ) );

    final DatastoreService ds = DatastoreServiceFactory.getDatastoreService();
       final Entity e;
    try {
        e = ds.get( key );
       } catch ( final EntityNotFoundException enfe ) {
        LOGGER.log( Level.WARNING,"Entity not found!",enfe );
        return;
       }

       if ( !e.getKind().equals( "Rep" ) && !e.getKind().equals( "Smpd" ) && !e.getKind().equals( "OtherFile" ) ) {
        LOGGER.severe( "Invalid Entity kind:" + e.getKind() );
        return;
       }

       if ( (Long) e.getProperty( "v" ) == 4 ) {
        LOGGER.warning( "Entity is already v4!" );
        return;
    }

       // Update common properties:
       // Todo
       final int size = ( (Long) e.getProperty( "size" ) ).intValue();
       if ( size < FileServlet.DATASTORE_CONTENT_STORE_LIMIT ) {
           final FileService fileService = FileServiceFactory.getFileService();
        final AppEngineFile   appeFile = new AppEngineFile( FileSystem.BLOBSTORE,( (BlobKey) e.getProperty( "blobKey" ) ).getKeyString() );
        final FileReadChannel channel  = fileService.openReadChannel( appeFile,false );
        final byte[]          content  = new byte[ size ];
        final ByteBuffer      wrapper  = ByteBuffer.wrap( content );
        while ( channel.read( wrapper ) > 0 )
            ;
        channel.close();

        e.setProperty( "content",new Blob( content ) );
        e.setProperty( "blobKey",null );
        fileService.delete( appeFile );
       }
       e.setUnindexedProperty( "blobKey",e.getProperty( "blobKey" ) );
       e.setUnindexedProperty( "content",e.getProperty( "content" ) );

       switch ( fileType ) {
       case SC2REPLAY :
           e.setUnindexedProperty( "matchup",e.getProperty( "matchup" ) );
        break;
       case MOUSE_PRINT :
        break;
       case OTHER :
        break;
       default:
        throw new RuntimeException( "Invalid file type!" );
       }

       // UPGRADE COMPLETE!
    e.setProperty( "v",4 );
    ds.put( e );
}

/**
 * List all entities.
 *
 * @param cursor used for pagination to determine which page to return
 * @param limit  the maximum number of entries to return
 * @return a response that encapsulates the result list and the next page token/cursor
 */
@ApiMethod(
        name = "list",path = "user",httpMethod = ApiMethod.HttpMethod.GET)
public CollectionResponse<User> list(@Nullable @Named("cursor") String cursor,@Nullable @Named("limit") Integer limit) {
    limit = limit == null ? DEFAULT_LIST_LIMIT : limit;
    Query<User> query = ofy().load().type(User.class).limit(limit);
    if (cursor != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursor));
    }
    QueryResultIterator<User> queryIterator = query.iterator();
    List<User> userList = new ArrayList<User>(limit);
    while (queryIterator.hasNext()) {
        userList.add(queryIterator.next());
    }
    return CollectionResponse.<User>builder().setItems(userList).setNextPagetoken(queryIterator.getCursor().toWebSafeString()).build();
}

/**
 * List all entities.
 *
 * @param cursor used for pagination to determine which page to return
 * @param limit  the maximum number of entries to return
 * @return a response that encapsulates the result list and the next page token/cursor
 */
@ApiMethod(
        name = "list",path = "joke",httpMethod = ApiMethod.HttpMethod.GET)
public CollectionResponse<Joke> list(@Nullable @Named("cursor") String cursor,@Nullable @Named("limit") Integer limit) {
    Integer limitParam = limit == null ? DEFAULT_LIST_LIMIT : limit;
    Query<Joke> query = ofy().load().type(Joke.class).limit(limitParam);
    if (cursor != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursor));
    }
    QueryResultIterator<Joke> queryIterator = query.iterator();
    List<Joke> jokeList = new ArrayList<>(limitParam);
    while (queryIterator.hasNext()) {
        jokeList.add(queryIterator.next());
    }
    return CollectionResponse.<Joke>builder().setItems(jokeList).setNextPagetoken(queryIterator.getCursor().toWebSafeString()).build();
}

/**
 * List all entities.
 *
 * @param cursor used for pagination to determine which page to return
 * @param limit  the maximum number of entries to return
 * @return a response that encapsulates the result list and the next page token/cursor
 */
@ApiMethod(
        name = "list",path = "orders",httpMethod = ApiMethod.HttpMethod.GET)
public CollectionResponse<Orders> list(@Nullable @Named("cursor") String cursor,@Nullable @Named("limit") Integer limit) {
    limit = limit == null ? DEFAULT_LIST_LIMIT : limit;
    Query<Orders> query = ofy().load().type(Orders.class).limit(limit);
    if (cursor != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursor));
    }
    QueryResultIterator<Orders> queryIterator = query.iterator();
    List<Orders> ordersList = new ArrayList<Orders>(limit);
    while (queryIterator.hasNext()) {
        ordersList.add(queryIterator.next());
    }
    return CollectionResponse.<Orders>builder().setItems(ordersList).setNextPagetoken(queryIterator.getCursor().toWebSafeString()).build();
}

/**
 * List all entities.
 *
 * @param cursor used for pagination to determine which page to return
 * @param limit  the maximum number of entries to return
 * @return a response that encapsulates the result list and the next page token/cursor
 */
@ApiMethod(
        name = "list",path = "users",httpMethod = ApiMethod.HttpMethod.GET)
public CollectionResponse<Users> list(@Nullable @Named("cursor") String cursor,@Nullable @Named("limit") Integer limit) {
    limit = limit == null ? DEFAULT_LIST_LIMIT : limit;
    Query<Users> query = ofy().load().type(Users.class).limit(limit);
    if (cursor != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursor));
    }
    QueryResultIterator<Users> queryIterator = query.iterator();
    List<Users> usersList = new ArrayList<Users>(limit);
    while (queryIterator.hasNext()) {
        usersList.add(queryIterator.next());
    }
    return CollectionResponse.<Users>builder().setItems(usersList).setNextPagetoken(queryIterator.getCursor().toWebSafeString()).build();
}

@Override
public Result<Book> listBooks(String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      .addSort(Book.TITLE,SortDirection.ASCENDING); // Use default Index "title"
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks,cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}

@Override
public Result<Book> listBooksByUser(String userId,String startCursorString) {
  FetchOptions fetchOptions = FetchOptions.Builder.withLimit(10); // Only show 10 at a time
  if (startCursorString != null && !startCursorString.equals("")) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(startCursorString)); // Where we left off
  }
  Query query = new Query(BOOK_KIND) // We only care about Books
      // Only for this user
      .setFilter(new Query.FilterPredicate(
          Book.CREATED_BY_ID,Query.FilterOperator.EQUAL,userId))
      // a custom datastore index is required since you are filtering by one property
      // but ordering by another
      .addSort(Book.TITLE,SortDirection.ASCENDING);
  PreparedQuery preparedQuery = datastore.prepare(query);
  QueryResultIterator<Entity> results = preparedQuery.asQueryResultIterator(fetchOptions);

  List<Book> resultBooks = entitiesToBooks(results);     // Retrieve and convert Entities
  Cursor cursor = results.getCursor();              // Where to start next time
  if (cursor != null && resultBooks.size() == 10) {         // Are we paging? Save Cursor
    String cursorString = cursor.toWebSafeString();               // Cursors are WebSafe
    return new Result<>(resultBooks,cursorString);
  } else {
    return new Result<>(resultBooks);
  }
}

/**
 * Counts the entities returned by the specified query.
 * @param ds    reference to the datastore service
 * @param query query whose results to count
 * @return the number of entities returned by the query
 */
public static int countEntities( final DatastoreService ds,final com.google.appengine.api.datastore.Query q ) {
    q.setKeysOnly();

    final int          batchSize    = 1000;
    final FetchOptions fetchOptions = FetchOptions.Builder.withLimit( batchSize );

    Cursor cursor = null;
    int    count  = 0;
    while ( true ) {
        if ( cursor != null )
            fetchOptions.startCursor( cursor );

        final QueryResultList< Entity > resultList = ds.prepare( q ).asQueryResultList( fetchOptions );

        count += resultList.size();

        if ( resultList.size() < batchSize )
            return count;

        cursor = resultList.getCursor();
    }
}

@Test
public void testCursor() throws Exception {
    Entity parent = createTestEntityWithUniqueMethodNameKey("Person","testKeysOnly");
    Key key = parent.getKey();

    Entity john = createEntity("Person",key)
        .withProperty("name","John")
        .withProperty("surname","Doe")
        .store();

    Query query = new Query("Person")
        .setAncestor(key)
        .setKeysOnly();

    PreparedQuery preparedQuery = service.prepare(query);
    QueryResultIterator<Entity> iter = preparedQuery.asQueryResultIterator();
    Assert.assertNotNull(iter.next());
    Cursor cursor = iter.getCursor();

    iter = service.prepare(query).asQueryResultIterator(FetchOptions.Builder.withStartCursor(cursor));
    Assert.assertFalse(iter.hasNext());
}

@Test
public void testSort() {
    int onePage = 6;
    Query query = new Query(kindName,rootKey);
    query.addSort("name",Query.sortDirection.ASCENDING);
    // fetch first page   aa,aa,aa
    Cursor cursor = checkPage(query,null,onePage,testDat[0],testDat[0]);
    Cursor decodedCursor = Cursor.fromWebSafeString(cursor.toWebSafeString());
    // fetch next page    aa,bb,bb
    checkPage(query,decodedCursor,testDat[1]);

    // desc
    onePage = total / testDat.length;
    query = new Query(kindName,Query.sortDirection.DESCENDING);
    // fetch first page   jj,jj,........,jj
    String chkChar = testDat[testDat.length - 1];
    cursor = checkPage(query,chkChar,chkChar);
    decodedCursor = Cursor.fromWebSafeString(cursor.toWebSafeString());
    // fetch next page   ii,ii,ii
    chkChar = testDat[testDat.length - 2];
    checkPage(query,chkChar);
}

@Test
public void testStartEndCursor() {
    int limit = total / testDat.length;
    Query query = new Query(kindName,Query.sortDirection.ASCENDING);
    FetchOptions fetchOption = FetchOptions.Builder.withLimit(limit);
    // fetch 1st page and get cursor1
    QueryResultList<Entity> nextBatch = service.prepare(query)
        .asQueryResultList(fetchOption);
    Cursor cursor1 = Cursor.fromWebSafeString(nextBatch.getCursor().toWebSafeString());
    // fetch 2nd page and get cursor2
    nextBatch = service.prepare(query).asQueryResultList(fetchOption.startCursor(cursor1));
    Cursor cursor2 = Cursor.fromWebSafeString(nextBatch.getCursor().toWebSafeString());
    // cursor1 as start and cursor2 as end and 15 in limit -- -- should return 2nd page.
    checkPage(query,cursor1,cursor2,limit,testDat[1],testDat[1]);
    // cursor1 as start and cursor2 as end and 30 in limit -- should return 2nd page.
    checkPage(query,2 * limit,testDat[1]);
    // cursor2 as start and cursor1 as end and 15 in limit -- should not return any.
    checkPage(query,null);
}

private Cursor checkPage(Query query,Cursor stCursor,Cursor endCursor,int limit,int exptRet,String chkSt,String chkEnd) {
    FetchOptions fetchOption = FetchOptions.Builder.withLimit(limit);
    if (stCursor != null) {
        fetchOption = fetchOption.startCursor(stCursor);
    }
    if (endCursor != null) {
        fetchOption = fetchOption.endCursor(endCursor);
    }
    QueryResultList<Entity> nextBatch = service.prepare(query)
        .asQueryResultList(fetchOption);
    assertEquals(exptRet,nextBatch.size());
    if (chkSt != null) {
        assertEquals(chkSt,nextBatch.get(0).getProperty("name"));
    }
    if (chkEnd != null) {
        assertEquals(chkEnd,nextBatch.get(nextBatch.size() - 1).getProperty("name"));
    }
    return nextBatch.getCursor();
}

@Nullable
public static PagingInfo create(@Nullable Cursor afterCursor,boolean moreResults,UriInfo uriInfo) {
  String afterCursorStr = (afterCursor == null) ? "" : afterCursor.toWebSafeString();
  if (afterCursorStr.isEmpty()) {
    return null;
  } else {
    String nextUrl;
    if (moreResults) {
      Multimap<String,String> queryParams = toMultimap(uriInfo.getQueryParameters());
      queryParams.replaceValues(AFTER_CURSOR_ParaM,asList(afterCursorStr));
      queryParams.replaceValues(LIMIT_ParaM,asList(String.valueOf(limit)));
      nextUrl = URLUtil.buildURL(uriInfo.getAbsolutePath(),queryParams);
    } else {
      nextUrl = null;
    }
    return new PagingInfo(nextUrl,afterCursorStr);
  }
}

private Builder(Class<T> resourceClass,@Nullable UriInfo uriInfo,@Nullable MultivaluedMap<String,String> queryParams,int defaultLimit) {
  if (uriInfo != null) {
    queryParams = uriInfo.getQueryParameters();
  }
  this.resourceClass = resourceClass;
  this.uriInfo = uriInfo;
  String afterCursorStr = queryParams.getFirst(PagingInfo.AFTER_CURSOR_ParaM);
  if (afterCursorStr != null) {
    afterCursor = Cursor.fromWebSafeString(afterCursorStr);
  } else {
    afterCursor = null;
  }
  limit = queryParams.containsKey(PagingInfo.LIMIT_ParaM) ?
      Integer.valueOf(queryParams.getFirst(PagingInfo.LIMIT_ParaM)) : defaultLimit;
  if (limit <= 0) {
    throw ErrorResponseMsg.createException("limit must be greater than zero",ErrorInfo.Type.BAD_REQUEST);
  }
}

@Override public Cursor getCursor() throws PermanentFailure,RetryableFailure {
  return safeRun(new Evaluater<Cursor>() {
    @Override public Cursor run() {
      return iterator.getCursor();
    }
  });
}

@ApiMethod(name = "listUsers")
public CollectionResponse<Users> listQuote(@Nullable @com.google.api.server.spi.config.Named("cursor") String cursorString,@Nullable @com.google.api.server.spi.config.Named("count") Integer count) {

    Query<Users> query = ofy().load().type(Users.class);
    if (count != null) query.limit(count);
    if (cursorString != null && cursorString != "") {
        query = query.startAt(Cursor.fromWebSafeString(cursorString));
    }
    List<Users> records = new ArrayList<Users>();
    QueryResultIterator<Users> iterator = query.iterator();
    int num = 0;
    while (iterator.hasNext()) {
        records.add(iterator.next());
        if (count != null) {
            num++;
            if (num == count) break;
        }
    }
    //Find the next cursor
    if (cursorString != null && cursorString != "") {
        Cursor cursor = iterator.getCursor();
        if (cursor != null) {
            cursorString = cursor.toWebSafeString();
        }
    }
    return CollectionResponse.<Users>builder().setItems(records).setNextPagetoken(cursorString).build();
}

@ApiMethod(
        name = "orderlist",httpMethod = ApiMethod.HttpMethod.GET)
public List<Orders> orderlist(@javax.annotation.Nullable @Named("cursor") String cursor){
    Query<Orders> query = ofy().load().type(Orders.class).limit(1000);
    if (cursor != null)
        query = query.startAt(Cursor.fromWebSafeString(cursor));
    QueryResultIterator<Orders> queryIterator = query.iterator();
    List<Orders> ordersList = new ArrayList<Orders>(1000);
    while (queryIterator.hasNext()) {
        ordersList.add(queryIterator.next());
    }
    return ordersList;
}

/**
 * Sets a cursor to a query so the next execute() method will return results after the last query
 * @param query           query to set the cursor of
 * @param lastQueryResult reference to the last query result list
 */
public static void setQueryCursor( final Query query,final List< ? > lastQueryResult ) {
    final Cursor cursor = JDOCursorHelper.getCursor( lastQueryResult );

    final Map< String,Object > extensionMap = new HashMap< String,Object >( 2 ); // initial size of 2 because 1 with default load factor=0.75 would result in 0-size internal cache...
    extensionMap.put( JDOCursorHelper.CURSOR_EXTENSION,cursor );

    query.setExtensions( extensionMap );
}

/**
 * Sets the page info to a query.
 * @param query    query to set the page info of
 * @param pageInfo page info to be set
 */
public static void setQueryPageInfo( final Query query,final PageInfo pageInfo ) {
    if ( pageInfo.getCursorString() == null )
        query.setRange( pageInfo.getoffset(),pageInfo.getoffset() + pageInfo.getLimit() );
    else {
        query.setRange( 0,pageInfo.getLimit() );

        final Map< String,Object >( 2 ); // initial size of 2 because 1 with default load factor=0.75 would result in 0-size internal cache...
        extensionMap.put( JDOCursorHelper.CURSOR_EXTENSION,Cursor.fromWebSafeString( pageInfo.getCursorString() ) );

        query.setExtensions( extensionMap );
    }
}

public SynchroForumCommand(Cursor p_cursor,int p_accountProcessed,int p_activeAccount,double p_maxLevel)
{
  m_cursor = p_cursor;
  m_accountProcessed = p_accountProcessed;
  m_activeAccount = p_activeAccount;
  m_maxLevel = p_maxLevel;
}

public static List<UserJDO> listAllUsers(PersistenceManager pm,String cursorString) {
    javax.jdo.Query query = pm.newQuery(UserJDO.class);
    if (cursorString != null) {
        Cursor cursor = Cursor.fromWebSafeString(cursorString);
        Map<String,Object> extensionMap = new HashMap<String,Object>();
        extensionMap.put(JDOCursorHelper.CURSOR_EXTENSION,cursor);
        query.setExtensions(extensionMap);
    }
    query.setRange(0,LIMIT);
    return (List<UserJDO>) query.execute();
}

public static List<RunJDO> listAllRuns(PersistenceManager pm,String cursorString) {
    javax.jdo.Query query = pm.newQuery(RunJDO.class);
    if (cursorString != null) {
        Cursor cursor = Cursor.fromWebSafeString(cursorString);
        Map<String,LIMIT);
    return (List<RunJDO>) query.execute();
}

@Override
public Pair<? extends Iterable<JobRecord>,String> queryRootPipelines(String classFilter,String cursor,final int limit) {
  Query query = new Query(JobRecord.DATA_STORE_KIND);
  Filter filter = classFilter == null || classFilter.isEmpty() ? new FilterPredicate(
      ROOT_JOB_disPLAY_NAME,GREATER_THAN,null)
      : new FilterPredicate(ROOT_JOB_disPLAY_NAME,EQUAL,classFilter);
  query.setFilter(filter);
  final PreparedQuery preparedQuery = dataStore.prepare(query);
  final FetchOptions fetchOptions = FetchOptions.Builder.withDefaults();
  if (limit > 0) {
    fetchOptions.limit(limit + 1);
  }
  if (cursor != null) {
    fetchOptions.startCursor(Cursor.fromWebSafeString(cursor));
  }
  return tryFiveTimes(
      new Operation<Pair<? extends Iterable<JobRecord>,String>>("queryRootPipelines") {
        @Override
        public Pair<? extends Iterable<JobRecord>,String> call() {
          QueryResultIterator<Entity> entities =
              preparedQuery.asQueryResultIterable(fetchOptions).iterator();
          Cursor dsCursor = null;
          List<JobRecord> roots = new LinkedList<>();
          while (entities.hasNext()) {
            if (limit > 0 && roots.size() >= limit) {
              dsCursor = entities.getCursor();
              break;
            }
            JobRecord jobRecord = new JobRecord(entities.next());
            roots.add(jobRecord);
          }
          return Pair.of(roots,dsCursor == null ? null : dsCursor.toWebSafeString());
        }
      });
}

public void setResult(final QueryResultList<Entity> result,final Cursor cursor,final boolean noMore) {
    final ArrayList<Entity> list = new ArrayList<>(result);

    if (bshFilter != null) {
        filterList(list);
    }

    resultBytes = SerializationHelper.getBytes(list);
    hasMore = !noMore;
    webCursor = cursor.toWebSafeString();
}

/**
 * Deletes all device subscription entities continuously using task push queue.
 *
 * @param time Threshold time before which entities created will be deleted. If time is null,*             current time is used and set as Threshold time.
 * @param cursor Query cursor indicates last query result set position
 */
protected void deleteallContinuously(Date time,String cursor) {
  if (time == null) {
    time = Calendar.getInstance(TimeZone.getTimeZone("UTC")).getTime();
  }

  Query.FilterPredicate timeFilter = new Query.FilterPredicate(PROPERTY_TIMESTAMP,FilterOperator.LESS_THAN_OR_EQUAL,time);
  QueryResultIterable<Entity> entities;
  List<Key> keys = new ArrayList<Key> ();
  List<String> subIds = new ArrayList<String> ();
  Query queryAll;

  queryAll = new Query(DeviceSubscription.SUBSCRIPTION_KIND).setFilter(timeFilter);
  FetchOptions options = FetchOptions.Builder.withLimit(BATCH_DELETE_SIZE);
  if (!StringUtility.isNullOrEmpty(cursor)) {
    options.startCursor(Cursor.fromWebSafeString(cursor));
  }

  entities = this.datastoreService.prepare(queryAll).asQueryResultIterable(options);
  if (entities != null && entities.iterator() != null) {
    for (Entity entity : entities) {
      keys.add(entity.getKey());
      String[] ids = new Gson().fromJson((String) entity.getProperty(PROPERTY_SUBSCRIPTION_IDS),String[].class);
      subIds.addAll(Arrays.asList(ids));
    }
  }

  if (keys.size() > 0) {
    deleteInBatch(keys);
    enqueueDeleteDeviceSubscription(time,entities.iterator().getCursor().toWebSafeString());
  }
  if (subIds.size() > 0) {
    deletePsiSubscriptions(subIds);
  }
}

private String merge(String cursorString) {
    Query<UserAnswer> query = ofy().load().type(UserAnswer.class).limit(1000);
    List<UserAnswer> toSaveList = new ArrayList<UserAnswer>(); 

    if (cursorString != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursorString));
    }

    boolean cont = false;
    QueryResultIterator<UserAnswer> iterator = query.iterator();

    while (iterator.hasNext()) {
        UserAnswer userAnswer = iterator.next();
        Map<String,String> answers = userAnswer.getAnswers();
        if(answers != null) {
            String answer = answers.get("householdIncome");
            if("$100,000-$149,999".equals(answer) || "$150,000-$199,999".equals(answer) ||
                    "$200,000-$249,999".equals(answer) || "$300,000 or more".equals(answer)) {
                answers.put("householdIncome","$100,000 or more");
                toSaveList.add(userAnswer);
            }
        }
        cont = true;
    }

    if(toSaveList.size() > 0) {
        ofy().save().entities(toSaveList).Now();
        logger.info(String.format("Merged %d answers",toSaveList.size()));
    }

    if(cont) {
        Cursor cursor = iterator.getCursor();
        return cursor.toWebSafeString();
    } else {
        return null;
    }
}

private String approve(String cursorString) {
    Query<UserAnswer> query = ofy().load().type(UserAnswer.class).limit(30);

    if (cursorString != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursorString));
    }

    boolean cont = false;
    QueryResultIterator<UserAnswer> iterator = query.iterator();

    while (iterator.hasNext()) {
        UserAnswer userAnswer = iterator.next();
        try {
            List<Assignment> assignments = getAssignmentsForHITService.getAssignments(true,userAnswer.getHitId());
            for(Assignment assignment: assignments) {
                approveAssignmentService.approveAssignment(true,assignment.getAssignmentId());
            }
        } catch (MturkException e) {
            logger.log(Level.WARNING,e.getMessage());
        }
        cont = true;
    }

    if(cont) {
        Cursor cursor = iterator.getCursor();
        return cursor.toWebSafeString();
    } else {
        return null;
    }
}

private String dispose(String cursorString) {
    Calendar endCal = Calendar.getInstance();
    endCal.setTime(new Date());
    endCal.set(Calendar.HOUR_OF_DAY,0);
    endCal.set(Calendar.MINUTE,0);
    endCal.set(Calendar.SECOND,0);

    Calendar startCal = Calendar.getInstance();
    startCal.setTime(endCal.getTime());
    startCal.add(Calendar.DAY_OF_MONTH,-1);

    Query<UserAnswer> query = ofy().load().type(UserAnswer.class)
            .filter("date >=",startCal.getTime()).filter("date <",endCal.getTime()).limit(30);

    if (cursorString != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursorString));
    }

    boolean cont = false;
    QueryResultIterator<UserAnswer> iterator = query.iterator();

    while (iterator.hasNext()) {
        UserAnswer userAnswer = iterator.next();
        try {
            disposeHITService.disposeHIT(true,userAnswer.getHitId());
            logger.log(Level.INFO,String.format("disposed HIT %s",userAnswer.getHitId()));
        } catch (MturkException e) {
            logger.log(Level.WARNING,e.getMessage());
        }
        cont = true;
    }

    if(cont) {
        Cursor cursor = iterator.getCursor();
        return cursor.toWebSafeString();
    } else {
        return null;
    }
}

public CollectionResponse<UserAnswer> list(String cursorString,Integer limit) throws NoSuchAlgorithmException {
    List<UserAnswer> result = new ArrayList<UserAnswer>();
    Query<UserAnswer> query = ofy().load().type(UserAnswer.class)
            .filter("surveyId",DEmogRAPHICS_SURVEY_ID).order("-date");

    if(cursorString != null) {
        query = query.startAt(Cursor.fromWebSafeString(cursorString));
    }

    if(limit != null) {
        query = query.limit(limit);
    }

    boolean cont = false;
    QueryResultIterator<UserAnswer> iterator = query.iterator();

    while (iterator.hasNext()) {
        UserAnswer userAnswer = iterator.next();
        String workerId = userAnswer.getWorkerId();
        if(workerId != null) {
            userAnswer.setWorkerId(MD5.crypt(workerId));
        }
        userAnswer.setIp(null);
        result.add(userAnswer);
        cont = true;
    }

    if(cont) {
        Cursor cursor = iterator.getCursor();
        return CollectionResponse.<UserAnswer> builder().setItems(result).setNextPagetoken(cursor.toWebSafeString()).build();
    } else {
        return CollectionResponse.<UserAnswer> builder().setItems(result).build();
    }
}

/**
 * This method lists all the entities inserted in datastore.
 * It uses HTTP GET method and paging support.
 *
 * @return A CollectionResponse class containing the list of all entities
 * persisted and a cursor to the next page.
 */
@SuppressWarnings({ "unchecked","unused" })
@ApiMethod(name = "listCategory")
public CollectionResponse<Category> listCategory(
        @Nullable @Named("cursor") String cursorString,@Nullable @Named("limit") Integer limit) {

    EntityManager mgr = null;
    Cursor cursor = null;
    List<Category> execute = null;

    try {
        mgr = getEntityManager();
        Query query = mgr.createquery("select from Category as Category");
        if (cursorString != null && cursorString != "") {
            cursor = Cursor.fromWebSafeString(cursorString);
            query.setHint(JPACursorHelper.CURSOR_HINT,cursor);
        }

        if (limit != null) {
            query.setFirstResult(0);
            query.setMaxResults(limit);
        }

        execute = (List<Category>) query.getResultList();
        cursor = JPACursorHelper.getCursor(execute);
        if (cursor != null)
            cursorString = cursor.toWebSafeString();

        // Tight loop for fetching all entities from datastore and accomodate
        // for lazy fetch.
        for (Category obj : execute)
            ;
    } finally {
        mgr.close();
    }

    return CollectionResponse.<Category> builder().setItems(execute)
            .setNextPagetoken(cursorString).build();
}

/**
 * This method lists all the entities inserted in datastore.
 * It uses HTTP GET method and paging support.
 *
 * @return A CollectionResponse class containing the list of all entities
 * persisted and a cursor to the next page.
 */
@SuppressWarnings({ "unchecked","unused" })
@ApiMethod(name = "listExpense")
public CollectionResponse<Expense> listExpense(
        @Nullable @Named("cursor") String cursorString,@Nullable @Named("limit") Integer limit) {

    EntityManager mgr = null;
    Cursor cursor = null;
    List<Expense> execute = null;

    try {
        mgr = getEntityManager();
        Query query = mgr.createquery("select from Expense as Expense");
        if (cursorString != null && cursorString != "") {
            cursor = Cursor.fromWebSafeString(cursorString);
            query.setHint(JPACursorHelper.CURSOR_HINT,cursor);
        }

        if (limit != null) {
            query.setFirstResult(0);
            query.setMaxResults(limit);
        }

        execute = (List<Expense>) query.getResultList();
        cursor = JPACursorHelper.getCursor(execute);
        if (cursor != null)
            cursorString = cursor.toWebSafeString();

        // Tight loop for fetching all entities from datastore and accomodate
        // for lazy fetch.
        for (Expense obj : execute)
            ;
    } finally {
        mgr.close();
    }

    return CollectionResponse.<Expense> builder().setItems(execute)
            .setNextPagetoken(cursorString).build();
}

/**
 * This method lists all the entities inserted in datastore.
 * It uses HTTP GET method and paging support.
 *
 * @return A CollectionResponse class containing the list of all entities
 * persisted and a cursor to the next page.
 */
@SuppressWarnings({ "unchecked","unused" })
@ApiMethod(name = "listIncome")
public CollectionResponse<Income> listIncome(
        @Nullable @Named("cursor") String cursorString,@Nullable @Named("limit") Integer limit) {

    EntityManager mgr = null;
    Cursor cursor = null;
    List<Income> execute = null;

    try {
        mgr = getEntityManager();
        Query query = mgr.createquery("select from Income as Income");
        if (cursorString != null && cursorString != "") {
            cursor = Cursor.fromWebSafeString(cursorString);
            query.setHint(JPACursorHelper.CURSOR_HINT,cursor);
        }

        if (limit != null) {
            query.setFirstResult(0);
            query.setMaxResults(limit);
        }

        execute = (List<Income>) query.getResultList();
        cursor = JPACursorHelper.getCursor(execute);
        if (cursor != null)
            cursorString = cursor.toWebSafeString();

        // Tight loop for fetching all entities from datastore and accomodate
        // for lazy fetch.
        for (Income obj : execute)
            ;
    } finally {
        mgr.close();
    }

    return CollectionResponse.<Income> builder().setItems(execute)
            .setNextPagetoken(cursorString).build();
}

/**
 * Deletes all device subscription entities continuously using task push queue.
 *
 * @param time Threshold time before which entities created will be deleted. If time is null,entities.iterator().getCursor().toWebSafeString());
  }
  if (subIds.size() > 0) {
    deletePsiSubscriptions(subIds);
  }
}

/**
 * Deletes all device subscription entities continuously using task push queue.
 *
 * @param time Threshold time before which entities created will be deleted. If time is null,entities.iterator().getCursor().toWebSafeString());
  }
  if (subIds.size() > 0) {
    deletePsiSubscriptions(subIds);
  }
}

@Test
public void testStartCursor() {
    QueryResultList<Entity> results = executeQuery(withLimit(3));
    Cursor cursor = results.getCursor();    // points to foo4

    results = executeQuery(withStartCursor(cursor));
    assertEquals(asList(foo4,foo5),results);
}

@Test
public void testStartCursorAndLimit() {
    QueryResultList<Entity> results = executeQuery(withLimit(3));
    Cursor cursor = results.getCursor();    // points to foo4

    results = executeQuery(withStartCursor(cursor).limit(1));
    assertEquals(asList(foo4),results);
}

@Test
public void testStartCursorAndOffset() {
    QueryResultList<Entity> results = executeQuery(withLimit(3));
    Cursor cursor = results.getCursor();    // points to foo4

    results = executeQuery(withStartCursor(cursor).offset(1));
    assertEquals(asList(foo5),results);
}