Develop Reference

How to pre-populate an existing DB with data?

The app I'm building currently has a Workout database.
Workout DB has a Workout table and a WoroutSets table.
The data in these two tables is inserted(saved) through user input.
And that's where some data is stored.
By the way, I want to put pre-populated data called WorkoutList into this Workout DB.
I consulted the docs for this.
I exported Workout.db and pre-populated it with data in DB Browser for SQLite.
And we are going to use createFromAsset("Workout.db") as per the docs. (Haven't tried yet)
However, what I am concerned about is whether there is a conflict between the Work DB of the existing app and the Workout DB to which the WorkoutList table has been added.

Assuming that you want to preserve each app users workouts and workoutsetss that they have input then you would not want to overwrite them by using createFromAsset.
Rather I suspect that what you want to do is introduce a new workoutlist table that is populated with predefined/pre-existing rows in the workoutlist as per a database supplied as an asset. In this case you do not want to use the createFromAsset method (although you could potentially have a second database created from the asset, attach it to the original and then merge the data - this would be more complex than it need be).
You also have to consider how to handle new installs, in which case there will be no existing user input workouts and workoutsetss, in which case you could use createFromAsset method. However, you would not want any other user's workouts and workoutsetss rows.
Based upon this assumption perhaps consider this demo that introduces a new table (workoutlist) whose data is retrieved from an asset maintaining the original user data in the other tables (workout and workoutset) but for a new install of the App creates database from the asset.
the schema is made up so will very likely differ from yours but the principle applies.
Java has been used but it would take little to change it to Kotlin
Workout
#Entity
class Workout {
#PrimaryKey
Long workoutId=null;
String workoutName;
Workout(){};
#Ignore
Workout(String workoutName) {
this.workoutId=null;
this.workoutName=workoutName;
}
}
WorkoutSet (plural not used but easily changed)
#Entity
class WorkoutSet {
#PrimaryKey
Long workoutSetId=null;
String workoutSetName;
long workoutIdMap;
WorkoutSet(){}
#Ignore
WorkoutSet(String workoutSetName, long parentWorkout) {
this.workoutSetId=null;
this.workoutSetName = workoutSetName;
this.workoutIdMap = parentWorkout;
}
}
WorkkoutList (the new table)
#Entity
class WorkoutList {
#PrimaryKey
Long workoutListId=null;
String workoutListName;
}
AllDAO (just for completeness)
#Dao
abstract class AllDAO {
#Insert(onConflict = OnConflictStrategy.IGNORE)
abstract long insert(Workout workout);
#Insert(onConflict = OnConflictStrategy.IGNORE)
abstract long insert(WorkoutSet workoutSet);
#Query("SELECT count(*) FROM workout")
abstract long getNumberOfWorkouts();
}
WorkoutDatabase the #Database annotated class
#Database(entities = {Workout.class,WorkoutSet.class, WorkoutList.class /*<<<<<<<<<< ADDED for V2 */}, exportSchema = false, version = MainActivity.DATABASE_VERSION)
abstract class WorkoutDatabase extends RoomDatabase {
abstract AllDAO getAllDAO();
private static Context passed_context;
private static volatile WorkoutDatabase INSTANCE;
static WorkoutDatabase getInstance(Context context) {
passed_context = context;
if (INSTANCE==null && MainActivity.DATABASE_VERSION == 1) {
INSTANCE = Room.databaseBuilder(context,WorkoutDatabase.class,MainActivity.DATABASE_NAME)
.allowMainThreadQueries()
.build();
}
if (INSTANCE ==null && MainActivity.DATABASE_VERSION > 1) {
INSTANCE = Room.databaseBuilder(context,WorkoutDatabase.class,MainActivity.DATABASE_NAME)
.allowMainThreadQueries()
.createFromAsset(MainActivity.DATABASE_ASSET_NAME) /* so new App installs use asset */
.addMigrations(MIGRATION_1_TO_2) /* to handle migration */
.build();
}
return INSTANCE;
}
static Migration MIGRATION_1_TO_2 = new Migration(1,2) {
#SuppressLint("Range")
#Override
public void migrate(#NonNull SupportSQLiteDatabase database) {
/* Create the new table */
database.execSQL("CREATE TABLE IF NOT EXISTS `WorkoutList` (`workoutListId` INTEGER, `workoutListName` TEXT, PRIMARY KEY(`workoutListId`))");
/* Cater for copying the data from the asset */
String tempDBName = "temp_" + MainActivity.DATABASE_NAME; /* name of the temporary/working database NOT an SQLITE TEMP database */
String newTableName = "workoutlist"; /* The table name */
String qualifiedNewTableName = tempDBName + "." + newTableName; /* The fully qualified new table name for the attached temp/wrk db */
String tempDBPath = passed_context.getDatabasePath(MainActivity.DATABASE_NAME).getParent() + File.separator + tempDBName; /* path to temp/wrk db */
try {
/* Copy the asset to a second DB */
InputStream asset = passed_context.getAssets().open(MainActivity.DATABASE_ASSET_NAME); /* open the asset */
File tempDB_File = new File(tempDBPath); /* File for temp/wrk database */
OutputStream tempdb = new FileOutputStream(tempDB_File); /* now an output stream ready for the copy */
int bufferLength = 1024 * 8; /* length of buffer set to 8k */
byte[] buffer = new byte[bufferLength]; /* the buffer for the copy */
/* copy the temp/wrk database from the asset to it's location */
while(asset.read(buffer) > 0) {
tempdb.write(buffer);
}
/* clean up after copy */
tempdb.flush();
tempdb.close();
asset.close();
/*Use the temporary/working database to populate the actual database */
/* Issues with WAL file change because migration is called within a transaction as per */
/* java.lang.IllegalStateException: Write Ahead Logging (WAL) mode cannot be enabled or disabled while there are transactions in progress. .... */
/* SO COMMENTED OUT */
//database.execSQL("ATTACH DATABASE '" + tempDBPath + "' AS " + tempDBName);
//database.execSQL("INSERT INTO " + newTableName + " SELECT * FROM " + qualifiedNewTableName);
//database.execSQL("DETACH " + tempDBName);
/* Alternative to ATTACH */
SQLiteDatabase assetdb = SQLiteDatabase.openDatabase(tempDB_File.getPath(),null,SQLiteDatabase.OPEN_READONLY);
Cursor csr = assetdb.query(newTableName,null,null,null,null,null,null);
ContentValues cv = new ContentValues();
while (csr.moveToNext()) {
cv.clear();
for (String s: csr.getColumnNames()) {
cv.put(s,csr.getString(csr.getColumnIndex(s)));
}
database.insert(newTableName,SQLiteDatabase.CONFLICT_IGNORE,cv);
}
assetdb.close();
tempDB_File.delete(); /* delete the temporary/working copy of the asset */
} catch (Exception e) {
/* handle issues here e.g. no asset, unable to read/write an so on */
e.printStackTrace();
}
}
};
}
This has been written to allow easy switching/running of the App with either version, simply two changes to run as old or new version of the App.
to run as version 1 DATABASE_VERSION (in MainActivity) is set to 1
AND the WorkoutSet class is commented out in the #Database annotation.
the Mirgration handles the copy of the data from the asset if the database already exists, otherwise for a new file then createFromAssets is used to copy the database from the asset.
MainActivity the activity code that does something with the database to ensure that it is opened/accessed
public class MainActivity extends AppCompatActivity {
public static final String DATABASE_NAME = "workout.db";
public static final int DATABASE_VERSION = 2;
public static final String DATABASE_ASSET_NAME = "testit.db"/*DATABASE_NAME*/ /* could be different */;
WorkoutDatabase wdb;
AllDAO dao;
#Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
wdb = WorkoutDatabase.getInstance(this);
dao = wdb.getAllDAO();
if (dao.getNumberOfWorkouts() < 1) addInitialData();
//wdb.close(); // FORCE close so WAL is full checkpointed (i.e. no -wal or -shm)
}
private void addInitialData() {
String prefix = String.valueOf(System.currentTimeMillis()); // to differentiate devices
dao.insert(new Workout(prefix + "-W001"));
dao.insert(new Workout(prefix + "-W002"));
dao.insert(new WorkoutSet(prefix + "-WS001",1));
dao.insert(new WorkoutSet(prefix + "-WS002",2));
}
}
note the wdb.close() is uncommented when creating the database to be loaded into the SQlite tool.
testit.db the SQLite database file as modified to introduce the new workoutList table. This first copied from a run of the App at Version 1 and then adding the new table (according SQL copied from the createAllTables method of WorkoutDataabase_Impl class in the generated java (aka the exact table as per Room's expectations)).
Note really the rows should be deleted as they are user specific. Instead -WRONG has been added to the end of the data (helps to prove the some points)
as above
The new table
Navicat was the SQLite tool used rather than DB Browser.
Run 1 running at version 1 to populate the database
DATABASE_VERSION = 1
#Database(entities = {Workout.class,WorkoutSet.class/*, WorkoutList.class*/ /*<<<<<<<<<< ADDED for V2 */}, exportSchema = false, version = MainActivity.DATABASE_VERSION)
i.e. the WorkoutList table has been excluded
App Inspection shows:-
and
Impotantly no WorkoutList table
Run 2 version 2 and introduction of new workoutlist table
DATABASE_VERSION = 2
#Database(entities = {Workout.class,WorkoutSet.class, WorkoutList.class /*<<<<<<<<<< ADDED for V2 */}, exportSchema = false, version = MainActivity.DATABASE_VERSION)
i.e. WorkoutList now introduced
App Inspection shows:-
old rows retained (they do not have -WRONG as per the asset)
old rows retained
the new table, populated as per the asset
Run 3 new install (after App uninstalled) at VERSION 2
as an be seen the incorrectly left in the assets rows exist (obviously you would delete the rows in real life but leaving them in shows that they are from the asset rather than generated by the code in the activity)
likewise
Conclusion
So the new install of the App correctly copies the asset database via createFromAsset whilst if migrating only the new data in the workoutlist table is copied from the asset.

Is it possible to enable/ disable transformation on selected Room Database Dao column?

Currently, we have the following database table
#Entity(
tableName = "note"
)
public class Note {
#ColumnInfo(name = "body")
private String body;
public String getBody() {
return body;
}
public void setBody(String body) {
this.body = body;
}
}
The length of the body string, can be from 0 to a very large number.
In certain circumstance, we need to
Load the all notes into memory.
A LiveData which is able to inform observers, if there's any changes made in the SQLite note table.
We just need the first 256 characters of body. We do not need entire body. Loading entire body string for all notes might cause OutOfMemoryException.
We have the following Room Database Dao
#Dao
public abstract class NoteDao {
#Query("SELECT * FROM note")
public abstract LiveData<List<Note>> getAllNotes();
}
getAllNotes able to fulfill requirements (1) and (2), but not (3).
The following getAllNotesWithShortBody is a failed solution.
#Dao
public abstract class NoteDao {
#Query("SELECT * FROM note")
public abstract LiveData<List<Note>> getAllNotes();
#Query("SELECT * FROM note")
public abstract List<Note> getAllNotesSync();
public LiveData<List<Note>> getAllNotesWithShortBody() {
MutableLiveData<List<Note>> notesLiveData = new MutableLiveData<>();
//
// Problem 1: Still can cause OutOfMemoryException by loading
// List of notes with complete body string.
//
List<Note> notes = getAllNotesSync();
for (Note note : notes) {
String body = note.getBody();
// Extract first 256 characters from body string.
body = body.substring(0, Math.min(body.length(), 256));
note.setBody(body);
}
notesLiveData.postValue(notes);
//
// Problem 2: The returned LiveData unable to inform observers,
// if there's any changes made in the SQLite `note` table.
//
return notesLiveData;
}
}
I was wondering, is there any way to tell Room database Dao: Before returning List of Notes as LiveData, please perform transformation on every Note's body column, by trimming the string to maximum 256 characters?
Examining the source code generated by Room Dao
If we look at the source code generated by Room Dao
#Override
public LiveData<List<Note>> getAllNotes() {
final String _sql = "SELECT * FROM note";
final RoomSQLiteQuery _statement = RoomSQLiteQuery.acquire(_sql, 0);
...
...
final String _tmpBody;
_tmpBody = _cursor.getString(_cursorIndexOfBody);
_tmpPlainNote.setBody(_tmpBody);
It will be great, if there is a way to supply transformation function during runtime, so that we can have
final String _tmpBody;
_tmpBody = transform_function(_cursor.getString(_cursorIndexOfBody));
_tmpPlainNote.setBody(_tmpBody);
p/s Please do not counter recommend Paging library at this moment, as some of our features require entire List of Notes (with trimmed body String) in memory.

You can use SUBSTR, one of SQLite's built-in functions.
You need a primary key in your #Entity. Assuming that you call it id, you can write a SQL like below.
#Query("SELECT id, SUBSTR(body, 0, 257) AS body FROM note")
public abstract LiveData<List<Note>> getAllNotes();
This will return the body trimmed to 256 chars.
With that being said, you should consider segmenting your rows. If you have too many rows, they will eventually use up your memory at some point. Using Paging is one way to do it. You can also use LIMIT and OFFSET to manually go through segments of rows.

Why Room select query is not working via c++?

I create the database via Room and gave the path to C++ developers via JNI. All data are inserting from C++ side by using sqlpp, then i am executing SQL query and not getting fresh datas. If i kill app then start again, SQL query is returning all datas.
#Provides
#Singleton
LocalDatabase provideLocalDatabase(#DatabaseInfo String dbName, Context context) {
return Room.databaseBuilder(context, LocalDatabase.class, dbName)
.setJournalMode(RoomDatabase.JournalMode.WRITE_AHEAD_LOGGING)
.build();
}
MessageDao
#Dao
public abstract class MessageDao implements BaseDao<Message> {
#Query("SELECT * FROM messages")
public abstract Flowable<Message> getMessage();
}
Sending db path to C++
String uiDbPath = MyApp.applicationContext.getDatabasePath(AppConstants.DB_NAME).getAbsolutePath();
send_db_path_to_c_plus_plus(uiDbPath);
We are using JournalMode.WRITE_AHEAD_LOGGING mode, after inserting data to database i am checking the database to make sure all data are exits. As a result i am seeing all data, but select query is not returning. How can i solve the issue please help? Thanks in advance.
EDIT 1 C++ parts
int64_t insert(manager::db::connection & conn, int64_t conv_id, int64_t author_id, const std::string & data)
{
auto conn_guard = manager::db::make_connection_guard(conn);
{
LOG_TRACE("db::insert(in_msg) - BEGIN conv_id={0} author_id={1}", conv_id, author_id);
const auto tab_in_msg = manager::db::tables::InMsgs{};
auto tx = start_transaction(conn_guard.native());
conn_guard.native()(insert_into(tab_in_msg).set(tab_in_msg.conversationId = conv_id, tab_in_msg.authorId = author_id, tab_in_msg.data = data));
int64_t rowid = conn_guard.native().last_insert_id();
tx.commit();
LOG_TRACE("db::insert(in_msg) - END rowid={0}", rowid);
return rowid;
}
}

RxJava2 TestSubscriber assertEmpty always true

I am trying to test some RxJava code using TestSubsriber. The data is coming from Room ORM as Flowable<Task>.
This is how my DAO class looks right now.
#Dao
public interface TaskDao {
#Insert()
long insertTask(TaskEntity task);
#Delete
int deleteTask(TaskEntity task);
#Query("SELECT * FROM task_table WHERE status = :taskStatus ORDER BY created_at DESC")
Flowable<List<TaskEntity>> getAllTasks(String taskStatus);
#Query("SELECT * FROM task_table WHERE id = :taskId")
Flowable<TaskEntity> getTask(String taskId);
#Update()
int updateTask(TaskEntity updatedTask);
#Query("SELECT COUNT(*) FROM task_table")
int rowCount();
}
I am trying to test getTask() method. Here the exact test method that is failing.
#Test
public void getTask_getSavedTask_returnsTask() {
Long resp = mTaskDao.insertTask(taskEntity);
assertThat(resp, is(1L));
assertThat(mTaskDao.rowCount(), is(1));
Flowable<TaskEntity> response = mTaskDao.getTask(taskEntity.getTaskId());
TestSubscriber<TaskEntity> testSubscriber = new TestSubscriber<>();
response.subscribe(testSubscriber);
testSubscriber.assertNoErrors();
testSubscriber.assertValueCount(1);
}
This code snippet fails at testSubscriber.assertValueCount(1) method.
The model is getting saved in db since the returned value from insert call is > 1. Also the row count is getting increased.
I tried running the same kind of code from outside the test environment (from an activity) and it's working fine there.

Looks like your DAO runs asynchronously and the TestSubscriber doesn't receive an answer immediately. Change the test to this and it should work:
testSubscriber.awaitCount(1, BaseTestConsumer.TestWaitStrategy.SLEEP_1MS, 5000);
testSubscriber.assertNoTimeout();
testSubscriber.assertNoErrors();
testSubscriber.assertValueCount(1);

How can I get a value from LiveData?

I am using Room for the first time. I am having a look at the LiveData concept. I know that we can fetch records from DB into LiveData and haveObservers attached.
#Query("SELECT * FROM users")
<LiveData<List<TCUser>> getAll();
But I am performing sync in the background, where I need to fetch data from server and compare it with the data in RoomDatabase table called "users" and then either insert,update or delete from users table. How can I traverse the LiveData list before taking any action ? As it gives error if I put it in for loop.
OR should I not use LiveData for this scenario ?
I guess I need to call
<LiveData<List<TCUser>> getAll().getValue()
But is it the right thing to do ? Here is some more code to give idea as to what I am trying to do:
List<User>serverUsers: Is the data received from a response from an API
private void updateUsers(List<User> serverUsers) {
List<UserWithShifts> users = appDatabase.userDao().getAllUsers();
HashMap<String, User> ids = new HashMap();
HashMap<String, User> newIds = new HashMap();
if (users != null) {
for (UserWithShifts localUser : users) {
ids.put(localUser.user.getId(), localUser.user);
}
}
for (User serverUser : serverUsers) {
newIds.put(serverUser.getId(), serverUser);
if (!ids.containsKey(serverUser.getId())) {
saveShiftForUser(serverUser);
} else {
User existingUser = ids.get(serverUser.getId());
//If server data is newer than local
if (DateTimeUtils.isLaterThan(serverUser.getUpdatedAt(), existingUser.getUpdatedAt())) {
deleteEventsAndShifts(serverUser.getId());
saveShiftForUser(serverUser);
}
}
}
Where:
#Query("SELECT * FROM users")
List<UserWithShifts> getAllUsers();
Is the first line in updateUsers() the right way to fetch data from DB to process before inserting new ones or should it be instead
<LiveData<List<User>> getAll().getValue()
Thanks,

If I understand your architecture correctly, updateUsers is inside of an AsyncTask or similar.
This is my proposed approach, which involves tweaking your Dao for maximum effectiveness. You wrote a lot of code to make decisions you could ask your database to make.
This is also not tight or efficient code, but I hope it illustrates more effective use of these libraries.
Background thread (IntentService, AsyncTask, etc.):
/*
* assuming this method is executing on a background thread
*/
private void updateUsers(/* from API call */List<User> serverUsers) {
for(User serverUser : serverUsers){
switch(appDatabase.userDao().userExistsSynchronous(serverUser.getId())){
case 0: //doesn't exist
saveShiftForUser(serverUser);
case 1: //does exist
UserWithShifts localUser = appDatabase.userDao().getOldUserSynchronous(serverUser.getId(), serverUser.getUpdatedAt());
if(localUser != null){ //there is a record that's too old
deleteEventsAndShifts(serverUser.getId());
saveShiftForUser(serverUser);
}
default: //something happened, log an error
}
}
}
If running on the UI thread (Activity, Fragment, Service):
/*
* If you receive the IllegalStateException, try this code
*
* NOTE: This code is not well architected. I would recommend refactoring if you need to do this to make things more elegant.
*
* Also, RxJava is better suited to this use case than LiveData, but this may be easier for you to get started with
*/
private void updateUsers(/* from API call */List<User> serverUsers) {
for(User serverUser : serverUsers){
final LiveData<Integer> userExistsLiveData = appDatabase.userDao().userExists(serverUser.getId());
userExistsLiveData.observe(/*activity or fragment*/ context, exists -> {
userExistsLiveData.removeObservers(context); //call this so that this same code block isn't executed again. Remember, observers are fired when the result of the query changes.
switch(exists){
case 0: //doesn't exist
saveShiftForUser(serverUser);
case 1: //does exist
final LiveData<UserWithShifts> localUserLiveData = appDatabase.userDao().getOldUser(serverUser.getId(), serverUser.getUpdatedAt());
localUserLiveData.observe(/*activity or fragment*/ context, localUser -> { //this observer won't be called unless the local data is out of date
localUserLiveData.removeObservers(context); //call this so that this same code block isn't executed again. Remember, observers are fired when the result of the query changes.
deleteEventsAndShifts(serverUser.getId());
saveShiftForUser(serverUser);
});
default: //something happened, log an error
}
});
}
}
You'll want to modify the Dao for whatever approach you decide to use
#Dao
public interface UserDao{
/*
* LiveData should be chosen for most use cases as running on the main thread will result in the error described on the other method
*/
#Query("SELECT * FROM users")
LiveData<List<UserWithShifts>> getAllUsers();
/*
* If you attempt to call this method on the main thread, you will receive the following error:
*
* Caused by: java.lang.IllegalStateException: Cannot access database on the main thread since it may potentially lock the UI for a long periods of time.
* at android.arch.persistence.room.RoomDatabase.assertNotMainThread(AppDatabase.java:XXX)
* at android.arch.persistence.room.RoomDatabase.query(AppDatabase.java:XXX)
*
*/
#Query("SELECT * FROM users")
List<UserWithShifts> getAllUsersSynchronous();
#Query("SELECT EXISTS (SELECT * FROM users WHERE id = :id)")
LiveData<Integer> userExists(String id);
#Query("SELECT EXISTS (SELECT * FROM users WHERE id = :id)")
Integer userExistsSynchronous(String id);
#Query("SELECT * FROM users WHERE id = :id AND updatedAt < :updatedAt LIMIT 1")
LiveData<UserWithShifts> getOldUser(String id, Long updatedAt);
#Query("SELECT * FROM users WHERE id = :id AND updatedAt < :updatedAt LIMIT 1")
UserWithShifts getOldUserSynchronous(String id, Long updatedAt);
}
Does this solve your problem?
NOTE: I did not see your saveShiftForUser or deleteEventsAndShifts methods. Insert, Save and Update are performed synchronously by Room. If you are running either method on the main thread (I'm guessing this is where your error is coming from), you should create a daoWrapper that is returned from appDatabase like so:
public class UserDaoWrapper {
private final UserDao userDao;
public UserDaoWrapper(UserDao userDao) {
this.userDao = userDao;
}
public LiveData<Long[]> insertAsync(UserWithShifts... users){
final MutableLiveData<Long[]> keys = new MutableLiveData<>();
HandlerThread ht = new HandlerThread("");
ht.start();
Handler h = new Handler(ht.getLooper());
h.post(() -> keys.postValue(userDao.insert(users)));
return keys;
}
public void updateAsync(UserWithShifts...users){
HandlerThread ht = new HandlerThread("");
ht.start();
Handler h = new Handler(ht.getLooper());
h.post(() -> {
userDao.update(users);
});
}
public void deleteAsync(User... users){
HandlerThread ht = new HandlerThread("");
ht.start();
Handler h = new Handler(ht.getLooper());
h.post(() -> {
for(User e : users)
userDao.delete(e.getId());
});
}
}