Code Explanation

Server

1. Receiving Photos from Application

var _storage = multer.diskStorage({
  destination: function (req, file, cb) {
    cb(null, 'uploads/')
  },
  filename: function (req, file, cb) {
    var date = new Date();
    modifiedDate = date.yyyymmdd() + date.hhmmss();
    cb(null, modifiedDate + '-' + file.originalname);
  }
})
var upload = multer({ storage: _storage })
app.post('/upload', upload.single('upload'), function(req, res){
  console.log(req.file);
  res.send('Uploaded : '+req.file.filename);
  analyzer.quickstart('uploads/' + req.file.filename).catch(console.error);
  analyzer.detectWeb('uploads/' + req.file.filename,  req.file.filename, modifiedDate).catch(console.error);
});
app.use('/uploads', express.static('uploads'));

• Receives photo with ‘multer’ module.
• Saves photo under ‘uploads’ folder.

2. Analyzing photos with Google Vision API

async function detectStart(fileDir, fileName, datetime) {
  
    const vision = require('@google-cloud/vision');
    const client = new vision.ImageAnnotatorClient();
    const [result] = await client.labelDetection(fileDir);
    const labels = result.labelAnnotations;
    console.log('Labels:');

    if (counter < 20){
      labels.forEach(function(label){
          keywords[counter] = label.description
          counter++;
      });
    }

    detectWeb(fileDir, fileName, datetime)
}

async function detectWeb(fileDir, fileName, datetime) {

    const vision = require('@google-cloud/vision');
    const client = new vision.ImageAnnotatorClient();
    const [result] = await client.webDetection(fileDir);
    const webDetection = result.webDetection;

    if (webDetection.webEntities.length) {
      console.log(`Web entities found: ${webDetection.webEntities.length}`);
      webDetection.webEntities.forEach(webEntity => {
        console.log(`  Description: ${webEntity.description}`);
        console.log(`  Score: ${webEntity.score}`);

        if (counter < 20){
          keywords[counter] = webEntity.description;
          counter++;
        }
      });
    }

    if (webDetection.bestGuessLabels.length) {
      console.log(
        `Best guess labels found: ${webDetection.bestGuessLabels.length}`
      );
      webDetection.bestGuessLabels.forEach(label => {
        console.log(`  Label: ${label.label}`);

        if (counter < 20){
          keywords[counter] = label.label;
          counter++;
        }
      });
    }

    insertQuery(fileName, datetime);
  }

• Uses basic tagging and web detection of Google Vision API from Google Cloud Platform.
• With maximum 20 tags(keywords), file names of the photos are saved in the database by ‘insertQuery’ function.

3. Saving photos in the database

async function insertQuery(fileName, datetime){

  var query = "INSERT INTO NUtellerData ";
  query += "VALUES ('" + fileName + "', " + datetime +", '";

  keywords.forEach(function(keyword){
      if(keyword != ""){
        query += "" + keyword + ", ";
        console.log(keyword);
      }
  });
  query = query.slice(0, -1);
  query += "')";

  console.log(query);
  dbconn.query(query, function(err, records){
      if(err) throw err;

      console.log('query updated!');
  });

  counter = 0;
  query = '';
  keywords = new Array();
}

• Saves the photo’s file name, uploaded datetime, and tags(keywords) in the database.
• Initializes counter, query, and keywords array.

4. Receives request from NUGU AI speaker.

let response =
  {
    "version": "2.0",
    "resultCode": "OK",
    "output": {
      "isQueryUpdated": "True",
      "isQueryExisted": "True"
    }
  }

let tag = "";

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
app.post('/' ,function(req, res){
  if(req.body.action.parameters.TAG.value){
    console.log("Received Tag From NUGU: " + req.body.action.parameters.TAG.value);
    tag = req.body.action.parameters.TAG.value;
    var deleteQuery = "DELETE FROM NUtellerRequested";
    dbconn.query(deleteQuery, function(err, records){
      if(err) throw err;
      console.log("Requested Table Initialized");
      var insertQuery = "INSERT INTO NUtellerRequested SELECT * FROM NUtellerData WHERE labels LIKE UPPER('%" + tag + "%');"
      dbconn.query(insertQuery, function(err, records){
        if(err) throw err;
        console.log("Requested Table is Generated!");
      })
    });
  }
  res.json(response);
});

• Receives request from NUGU AI speaker with requested tag.
• Inserts data that including finding tag into the database that consisted with requested data.
• When making database finished, send response to NUGU AI speaker to say appropriate answer.

5. Sending requested data to client(application).

app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: true}));
app.get('/result', function(req, res){
  res.setHeader('Content-Type', 'application/json');
  var query = "SELECT * FROM NUtellerRequested";
  dbconn.query(query,function(err, records){
    if(err) throw err;
    res.json(records);
  });
});

• Loads data from the requested database, renders it with ‘GET’ method.
• This data includes file name, file uploaded datetime, and tags.

Android application

1. MainActivity.java

public class MainActivity extends AppCompatActivity {
    ImageView imgVwSelected;
    File tempSelectFile;

    Button btnImageSend;
    Button btnImageSelection;
    Button btnTakePic;
    Button btnImageDelete;
    static final int PICK_FROM_CAMERA = 1; // Take picture using camera
    static final int PICK_FROM_ALBUM = 2; // Select picture from gallery

• When users start this application, first page will be with MainAcitivity.java
• Declare needed buttons, imageview, file variable for temporary fie
• Actions for selecting picture will be divided into two, PICK_FROM_CAMERA and PICK_FROM_ALBUM

@Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.activity_main);

    imgVwSelected = findViewById(R.id.imgVwSelected);

• Selected image will be displayed in this image view

btnImageSend = findViewById(R.id.btnImageSend);
btnImageSend.setEnabled(false);
btnImageSend.setOnClickListener(new View.OnClickListener() {
    @Override
    public void onClick(View view) {
        FileUploadUtils.goSend(tempSelectFile);
        Toast.makeText(getApplicationContext(), "File Transfer - Success!: " + tempSelectFile.getAbsolutePath(), Toast.LENGTH_LONG).show();
    }
});

• Button to send selected picture to server
• Selected image will be send to server through FileUploadUtils.java

btnImageSelection = findViewById(R.id.btnImageSelection);
btnImageSelection.setOnClickListener(new View.OnClickListener() {
    @Override
    public void onClick(View view) {
        Intent intent = new Intent(Intent.ACTION_PICK);
        intent.setType(android.provider.MediaStore.Images.Media.CONTENT_TYPE);
        intent.setData(android.provider.MediaStore.Images.Media.EXTERNAL_CONTENT_URI);
        startActivityForResult(intent, PICK_FROM_ALBUM);
    }
});

• Button to select picture from gallery
• User's image gallery will be loaded with new intent page.

btnTakePic = findViewById(R.id.btnTakePic);
btnTakePic.setOnClickListener(new View.OnClickListener() {
    @Override
    public void onClick(View view) {
        sendTakePhotoIntent();
    }
});

// Method to take picture
private void sendTakePhotoIntent() {
    // Take picture through new intent
    Intent takePictureIntent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE);
    if (takePictureIntent.resolveActivity(getPackageManager()) != null) {
        startActivityForResult(takePictureIntent, PICK_FROM_CAMERA);
    }
}

• Button to take picture
• User's camera application will be loaded with new intent page using sendTakePhotoInente() method, which is defined below.

btnImageDelete = findViewById(R.id.btnImageDelete);
btnImageDelete.setOnClickListener(new View.OnClickListener() {
    @Override
    public void onClick(View view) {
        if (tempSelectFile.exists()) {
            if (tempSelectFile.delete()) {
                Toast.makeText(getApplicationContext(), "File Delecte - Success!: " + tempSelectFile.getAbsolutePath(), Toast.LENGTH_LONG).show();

            } else {
                Toast.makeText(getApplicationContext(), "File Delecte - Fail: " + tempSelectFile.getAbsolutePath(), Toast.LENGTH_LONG).show();
            }
        } else {
            Toast.makeText(getApplicationContext(), "File doesn't exist " + tempSelectFile.getAbsolutePath(), Toast.LENGTH_LONG).show();
        }
    }
});

• Button to delete selected picture
• When selected image's path exists, delete this file using delete() method.

public String getImageNameToUri(Uri data) {
    String[] proj = {MediaStore.Images.Media.DATA};
    Cursor cursor = managedQuery(data, proj, null, null, null);
    int coluum_index = cursor.getColumnIndexOrThrow(MediaStore.Images.Media.DATA);

    cursor.moveToFirst();

    String imgPath = cursor.getString(coluum_index);
    String imgName = imgPath.substring(imgPath.lastIndexOf("/") + 1);

    return imgName;
}

• Method to get file's name
• This method will be used when application send selected image with image's path to server.

@Override
protected void onActivityResult(int requestCode, int resultCode, Intent data) {
    super.onActivityResult(requestCode, resultCode, data);

    if (resultCode != RESULT_OK) return;
    switch (requestCode) {
        case PICK_FROM_CAMERA: { // Take picture
            Bundle extras = data.getExtras();
            Bitmap image = (Bitmap) extras.get("data");
            ((ImageView) findViewById(R.id.imgVwSelected)).setImageBitmap(image);

            try {
                // Save selected image temporarily and send to server
                String date = new SimpleDateFormat("yyyy_MM_dd_hh_mm_ss").format(new Date());
                tempSelectFile = new File(Environment.getExternalStorageDirectory() + "/" + date + ".jpg");
                // File name will be default directory's path and date taken picture
                OutputStream out = new FileOutputStream(tempSelectFile);
                // Lower image's quality to 85% for faster transferring
                image.compress(Bitmap.CompressFormat.JPEG, 85, out);

            } catch (IOException ioe) {
                ioe.printStackTrace();
            }
            btnImageSend.setEnabled(true);
            break;


        }

• 'take picture' button sends PICK_FROM_CAMERA variable to onActivitiyResult.
• Photo taken will be temporarily saved in bitmap and send to server with default directory's path and date.

case PICK_FROM_ALBUM: { // Get picture from gallery
                try {
                    // Get image's name from Uri
                    String name_Str = getImageNameToUri(data.getData());

                    // Get image date in bitmap
                    Bitmap image_bitmap = MediaStore.Images.Media.getBitmap(getContentResolver(), data.getData());
                    imgVwSelected.setImageBitmap(image_bitmap);

                    // Save selected image temporarily and send to server
                    tempSelectFile = new File(Environment.getExternalStorageDirectory() + "/" + name_Str);
                    // File name will be default directory's path and extracted file's name
                    OutputStream out = new FileOutputStream(tempSelectFile);
                    // Lower image's quality to 50% for faster transferring
                    image_bitmap.compress(Bitmap.CompressFormat.JPEG, 50, out);

                } catch (FileNotFoundException e) {
                    e.printStackTrace();
                } catch (IOException e) {
                    e.printStackTrace();
                } catch (Exception e) {
                    e.printStackTrace();
                }
                btnImageSend.setEnabled(true);
                break;
            }
        }
    }
}

• 'selection from gallery' button sends PICK_FROM_GALLERY variable to onActivitiyResult.
• Photo selected will be temporarily saved in bitmap and send to server with default directory's path and file's name.

2. FileUploadUtils.java

import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;

public class FileUploadUtils {
    public static void goSend(File file){

        RequestBody requestBody = new MultipartBody.Builder()
                .setType(MultipartBody.FORM)
                .addFormDataPart("upload", file.getName(),
                        RequestBody.create(MultipartBody.FORM, file))
                .build();

        Request request = new Request.Builder()
                .url("http://rpi-kyc.iptime.org:9999/upload")
                .post(requestBody)
                .build();

• Send connection request to server using post(default) method
• Use okhttp library to connection

OkHttpClient client = new OkHttpClient();

    client.newCall(request).enqueue(new Callback() {
        @Override
        public void onFailure(Call call, IOException e) {
            e.printStackTrace();
        }
        @Override
        public void onResponse(Call call, Response response) throws IOException {
            Log.d("TEST : ", response.body().string());
        }
    });
}

• Using enqueue, implement Callback for asynchronous processing

3. AndroidManifest.xml

<uses-permission
    android:name="android.permission.INTERNET"
    android:required="true" /> 
<uses-permission
    android:name="android.permission.WRITE_EXTERNAL_STORAGE"
    android:required="true" /> 
<uses-permission
    android:name="android.permission.READ_EXTERNAL_STORAGE"
    android:required="true" /> 
<uses-permission
    android:name="android.permission.CAMERA"
    android:required="true" /> 
<uses-permission
    android:name="android.hardware.camera2"
    android:required="true" /> 

• Grant smartphone's internet, write and read to external memory, camera permission to application.

4. buildgradle(app)

dependencies {
    implementation 'com.squareup.okhttp3:okhttp:3.4.2'
    implementation 'com.google.code.gson:gson:2.8.6'
}

• To use okhttp library and gson (Google's json service), add to dependencies

5. network_security_config.

<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
    <base-config cleartextTrafficPermitted="true" />
</network-security-config>

• Android 9 restricts access to http protocols and allow only https, so make networkSecurityConfig.xml file and register to AndroidManifest.xml