How to use images in Android SQLite that are larger than the limitations of a CursorWindow?
As per the question. Say I have an Image that is approx 3.5 MB in size, which could be saved as a blob, but couldn't be retrieved due to the 2Mb limitation of a CursorWindow?
Note
The question has been asked purely to show, contrary to responses saying that it cannot be done, that it can be done (albeit NOT RECOMMENDED), as this is a It’s OK to Ask and Answer Your Own Questions.
Solution 1:
Note
This is not recommended as it would still likely be quite inefficient in comparison to storing the path to an image file.
The obvious answer is to split the image up into manageable parts (chunks)
- (say 256k chunks (14 such chunks would hold approx 3.5Mb))
allowing the individual chunks to be assembled when required.
Simple Example
This example illustrates both storing, retrieving, assembling and displaying an image that would be too large (approx 3.5MB).
-
To make the core image easily available it has been placed into the assets folder and is then copied, by the App, to the App's data (data/data/myimages/) (rather than writing extra code, say use the camera).
-
two tables are used
- a table, named imagemaster, for the singular image data e.g. it's name and
- a second table, named imagechunk for the chunks each referencing the respective row in the imagemaster table.
The DatabaseHelper DBHelper.java :-
public class DBHelper extends SQLiteOpenHelper {
public static final String DBNAME = "mydb";
public static final int DBVERSION = 1;
public static final String TBL_IMAGEMASTER = "image_matser";
public static final String COL_IMAGEMASTER_ID = BaseColumns._ID;
public static final String COL_IMAGEMASTER_DESCRIPTION = "description";
public static final String COL_IMAGEMASTER_THUMBNAIL = "thumbnail";
public static final String TBL_IMAGECHUNK = "imagechunk";
public static final String COL_IMAGECHUNK_ID = BaseColumns._ID;
public static final String COL_IMAGECHUNK_OWNER = "owner";
public static final String COL_IMAGECHUNK_CHUNK = "chunk";
public static final String COL_IMAGECHUNK_CHUNKORDER = "chunkorder";
public static final String COL_IMAGECHUNK_CHUNKSIZE = "chunksize";
public static final int MAXIMUM_CHUNK_SIZE = 256 * 1024; // 256k chunks
// Some codes for when/if things go wrong
public static final long NOSUCHFILE = -2;
public static final long INPUT_ASSIGN_IO_ERROR = -3;
public static final long INPUT_READ_IO_ERROR = -4;
public static final long CHUNKCOUNTMISMATCH = -5;
SQLiteDatabase mDB;
public DBHelper(Context context) {
super(context, DBNAME, null, DBVERSION);
mDB = this.getWritableDatabase(); //Open the database
}
@Override
public void onCreate(SQLiteDatabase db) {
// The imagemaster table
String mImageMasterCrtSQL = "CREATE TABLE IF NOT EXISTS " + TBL_IMAGEMASTER + "(" +
COL_IMAGEMASTER_ID + " INTEGER PRIMARY KEY, " +
COL_IMAGEMASTER_DESCRIPTION + " TEXT UNIQUE, " +
COL_IMAGEMASTER_THUMBNAIL + " BLOB DEFAULT x'00' " +
")";
db.execSQL(mImageMasterCrtSQL);
// The imagechunk table
String mImageChunkCrtSQL = "CREATE TABLE IF NOT EXISTS " + TBL_IMAGECHUNK + "(" +
COL_IMAGECHUNK_ID + " INTEGER PRIMARY KEY, " +
COL_IMAGECHUNK_OWNER + " INTEGER REFERENCES " + TBL_IMAGEMASTER + "(" +
COL_IMAGEMASTER_ID +
")," +
COL_IMAGECHUNK_CHUNKORDER + " INTEGER, " +
COL_IMAGECHUNK_CHUNKSIZE + " INTEGER, " +
COL_IMAGECHUNK_CHUNK + " BLOB DEFAULT x'00'" +
")";
db.execSQL(mImageChunkCrtSQL);
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
}
// Need to turn on FOREIGN KEY support
@Override
public void onConfigure(SQLiteDatabase db) {
super.onConfigure(db);
db.setForeignKeyConstraintsEnabled(true);
}
// Add an imagemaster row (private as imagemaster row and imagechunk rows are to be added together)
private long addImageMaster(String description, byte[] thumbnail) {
ContentValues cv = new ContentValues();
cv.put(COL_IMAGEMASTER_DESCRIPTION,description);
if (thumbnail.length > 0) {
cv.put(COL_IMAGEMASTER_THUMBNAIL,thumbnail);
}
return mDB.insert(TBL_IMAGEMASTER,null,cv);
}
// Add an imagechunk row (private as imagemaster and imagechucks will be added togther)
private long addImageChunk(long owningImage, long order, byte[] image) {
ContentValues cv = new ContentValues();
cv.put(COL_IMAGECHUNK_OWNER,owningImage);
cv.put(COL_IMAGECHUNK_CHUNKORDER,order);
cv.put(COL_IMAGECHUNK_CHUNKSIZE,image.length);
cv.put(COL_IMAGECHUNK_CHUNK,image);
return mDB.insert(TBL_IMAGECHUNK,null,cv);
}
// Add imagemaster and all imagechunk rows from a File
public long storeImageFromFile(String description, File image, byte[] thumbnail, boolean printstacktrace) {
long rv = NOSUCHFILE;
long master_id;
if (!image.exists()) {
return rv;
}
//Get image info from file
long chunkcount = (image.length() / (long) MAXIMUM_CHUNK_SIZE);
if ((image.length() - (chunkcount * (long) MAXIMUM_CHUNK_SIZE)) > 0) {
chunkcount++;
}
// Add the image master row
rv = addImageMaster(description, thumbnail);
if (rv < 1) {
return rv;
}
master_id = rv;
// Prepare to save chunks
byte[] buffer = new byte[MAXIMUM_CHUCK_SIZE];
int currentchunk = 0;
int readlength = 0;
rv = INPUT_ASSIGN_IO_ERROR;
long chunksavedcount = 0;
try {
InputStream is = new FileInputStream(image);
rv = INPUT_READ_IO_ERROR;
while ((readlength = is.read(buffer)) > 0) {
if (readlength == MAXIMUM_CHUNK_SIZE) {
if (addImageChunk(master_id, currentchunk++, buffer) > 0) {
chunksavedcount++;
}
} else {
byte[] lastbuffer = new byte[readlength];
for (int i = 0; i < readlength; i++) {
lastbuffer[i] = buffer[i];
}
if (addImageChunk(master_id, currentchunk, lastbuffer) > 0) {
chunksavedcount++;
}
}
}
is.close();
} catch (IOException ioe) {
if (printstacktrace) {
ioe.printStackTrace();
}
return rv;
}
if (chunksavedcount != chunkcount) {
rv = CHUNKCOUNTMISMATCH;
}
return rv;
}
//Get the image as a byte array (could easily return a BitMap) according to the image description
public byte[] getAllChunksAsByteArray(String imageDescription) {
String column_chucksize_sum = "chuck_size_sum";
long master_id = -1;
int imagesize = 0;
//Stage 1 get the image master id according to the description
String[] columns = new String[]{COL_IMAGEMASTER_ID};
String whereclause = COL_IMAGEMASTER_DESCRIPTION + "=?";
String[] whereargs = new String[]{imageDescription};
Cursor csr = mDB.query(TBL_IMAGEMASTER,columns,whereclause,whereargs,null,null,null,null);
if (csr.moveToFirst()) {
master_id = csr.getLong(csr.getColumnIndex(COL_IMAGEMASTER_ID));
}
//If no such image then return empty byte array
if (master_id < 1) {
csr.close();
return new byte[0];
}
// Stage 2 get the total size of the image
columns = new String[]{"sum(" + COL_IMAGECHUNK_CHUNKSIZE + ") AS " + column_chucksize_sum};
whereclause = COL_IMAGECHUNK_OWNER + "=?";
whereargs = new String[]{String.valueOf(master_id)};
csr = mDB.query(TBL_IMAGECHUNK,columns,whereclause,whereargs,null,null,COL_IMAGECHUNK_CHUNKORDER + " ASC");
if (csr.moveToFirst()) {
imagesize = csr.getInt(csr.getColumnIndex(column_chucksize_sum));
}
//If no chunks or all chunks are empty return empty byte array
if (imagesize < 1) {
csr.close();
return new byte[0];
}
//Stage 3 combine all the chunks into a single byte array
columns = new String[]{COL_IMAGECHUNK_CHUNK, COL_IMAGECHUNK_CHUNKSIZE};
csr = mDB.query(TBL_IMAGECHUNK,columns,whereclause,whereargs,null,null,COL_IMAGECHUNK_CHUNKORDER + " ASC");
if (csr.getCount() < 1) {
csr.close();
return new byte[0];
}
int rv_offset = 0;
byte[] rv = new byte[imagesize];
while (csr.moveToNext()) {
int currentsize = csr.getInt(csr.getColumnIndex(COL_IMAGECHUNK_CHUNKSIZE));
byte[] thischunk = csr.getBlob(csr.getColumnIndex(COL_IMAGECHUNK_CHUNK));
for (int i = 0; i < thischunk.length; i++) {
rv[rv_offset + i] = thischunk[i];
}
rv_offset = rv_offset + currentsize;
}
csr.close();
return rv;
}
}
The Activity MainActivity.java
public class MainActivity extends AppCompatActivity {
DBHelper mDBHlpr; //The database helper
ImageView mMyImageView; //For displaying the image (initially nothing shown)
Button mTestIt; //Button that will retrieve the image from the DB and display it
String mSaveDirectory = "myimages"; //The directory in which to save the image file
byte[] extracted_image; //For the retrieved image
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
mMyImageView = this.findViewById(R.id.myimageview);
mTestIt = this.findViewById(R.id.testit);
mTestIt.setOnClickListener(new View.OnClickListener() {
@Override
public void onClick(View v) {
showimage(extracted_image); //<<<<<<<<<< extract the image and display it.
}
});
mDBHlpr = new DBHelper(this); //<<<<<<<<<< instantiate the Database Helper
String testfilename = "20141107 1924 SCC Bedroom.JPG"; //The file to get from the assets folder
String testdescription = "MyTestImage"; //The description to give the image
//1. copy the file from the assets folder e.g. akin to taking photo from camera
File testfile = new File(saveAssetAsFile(testfilename));
//2. Add the image and the chucks to the DB
mDBHlpr.storeImageFromFile(testdescription,testfile,new byte[]{0,1,2,3,4,5,6},true);
//3. Extract the rows and write them to the log
Cursor csr = mDBHlpr.getWritableDatabase().query(DBHelper.TBL_IMAGEMASTER,null,null,null,null,null,null);
DatabaseUtils.dumpCursor(csr);
csr = mDBHlpr.getWritableDatabase().query(DBHelper.TBL_IMAGECHUNK,null,null,null,null,null,null);
DatabaseUtils.dumpCursor(csr);
csr.close();
//4. extract the byte array for the image display the length of the byte array
extracted_image = mDBHlpr.getAllChunksAsByteArray(testdescription);
Log.d("EXTRACTED","The extracted image size is " + String.valueOf(extracted_image.length));
}
//Copy the asset to a file
private String saveAssetAsFile(String asset) {
//For ease use data/data/<package_name>/myimages to save the image as a file
//Note a bit of a cheat as getDatabasePath will return data/data/<package_name>/databases/xxx (or equivalent)
//GetDatabasepath available for all Android versions
String filepath = this.getDatabasePath("xxx").getParentFile().getParent() + File.separator + mSaveDirectory + File.separator + asset;
File savelocation = new File(filepath);
//If the file exists then no need to copy again so return
if (savelocation.exists()) return savelocation.getPath();
//Create the myimages directory if needed (will be required first run)
if (!savelocation.getParentFile().exists()) {
savelocation.getParentFile().mkdirs();
}
byte[] buffer = new byte[DBHelper.MAXIMUM_CHUNK_SIZE]; //Use 256k buffer as size is defined
int buffer_length;
try {
InputStream is = this.getAssets().open(asset);
FileOutputStream os = new FileOutputStream(savelocation);
while ((buffer_length = is.read(buffer)) > 0) {
os.write(buffer,0,buffer_length);
}
os.flush();
os.close();
is.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
return savelocation.getPath();
}
private void showimage(byte[] imagetoshow) {
Bitmap bmp = BitmapFactory.decodeByteArray(imagetoshow, 0, imagetoshow.length);
mMyImageView.setImageBitmap(bmp);
}
}
Result
When the App is run (no image) :-
After clicking the button :-
The image is extracted from the DB
The log :-
........
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: 13 {
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: _id=14
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: owner=1
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: chunkorder=13
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: chunksize=97210
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: chunk=<unprintable>
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: }
03-24 16:44:36.416 22859-22859/aaa.so55276671hiddenimages I/System.out: <<<<<
03-24 16:44:36.423 22859-22859/aaa.so55276671hiddenimages W/CursorWindow: Window is full: requested allocation 262144 bytes, free space 261532 bytes, window size 2097152 bytes
03-24 16:44:36.441 22859-22859/aaa.so55276671hiddenimages W/CursorWindow: Window is full: requested allocation 262144 bytes, free space 261532 bytes, window size 2097152 bytes
03-24 16:44:36.453 22859-22859/aaa.so55276671hiddenimages D/EXTRACTED: The extracted image size is 3505082
........... click the button
03-24 16:50:09.565 22859-22859/aaa.so55276671hiddenimages D/BEFOREEXTRACT: Button clicked so extracting image.
03-24 16:50:09.583 22859-22872/aaa.so55276671hiddenimages I/art: Background sticky concurrent mark sweep GC freed 1882(116KB) AllocSpace objects, 7(1631KB) LOS objects, 0% free, 65MB/65MB, paused 6.779ms total 17.678ms
03-24 16:50:09.765 22859-22859/aaa.so55276671hiddenimages D/AFTEREXTRACT: Finished extracting the image.
- Note full CursorWindow messages aren't failures, rather just saying that an attempt was made to add a row, but it was full.
- WARNING As can be seen 5 such images would take 1 sec to extract and display