Detection Of Leaf On Unpredictable Background

A project I have been working about for some time is a unsupervised leaf segmentation. The leaves are captured on a white or colored paper, and some of them has shadows. I want to

Solution 1:

Try this on...I'm using "grabCut" from the openCV lib. It's not perfect, but it might be a good start.

import cv2
import numpy as np
from matplotlib import pyplot as plt
import matplotlib
#%matplotlib inline #uncomment if in notebook

def mask_leaf(im_name, external_mask=None):

    im = cv2.imread(im_name)
    im = cv2.blur(im, (5,5))

    height, width = im.shape[:2]

    mask = np.ones(im.shape[:2], dtype=np.uint8) * 2 #start all possible background
    '''
    #from docs:
    0 GC_BGD defines an obvious background pixels.
    1 GC_FGD defines an obvious foreground (object) pixel.
    2 GC_PR_BGD defines a possible background pixel.
    3 GC_PR_FGD defines a possible foreground pixel.
    '''

    #2 circles are "drawn" on mask. a smaller centered one I assume all pixels are definite foreground. a bigger circle, probably foreground.
    r = 100
    cv2.circle(mask, (int(width/2.), int(height/2.)), 2*r, 3, -3) #possible fg
    #next 2 are greens...dark and bright to increase the number of fg pixels.
    mask[(im[:,:,0] < 45) & (im[:,:,1] > 55) & (im[:,:,2] < 55)] = 1  #dark green
    mask[(im[:,:,0] < 190) & (im[:,:,1] > 190) & (im[:,:,2] < 200)] = 1  #bright green
    mask[(im[:,:,0] > 200) & (im[:,:,1] > 200) & (im[:,:,2] > 200) & (mask != 1)] = 0 #pretty white

    cv2.circle(mask, (int(width/2.), int(height/2.)), r, 1, -3) #fg

    #if you pass in an external mask derived from some other operation it is factored in here.
    if external_mask is not None:
        mask[external_mask == 1] = 1

    bgdmodel = np.zeros((1,65), np.float64)
    fgdmodel = np.zeros((1,65), np.float64)
    cv2.grabCut(im, mask, None, bgdmodel, fgdmodel, 1, cv2.GC_INIT_WITH_MASK)

    #show mask
    plt.figure(figsize=(10,10))
    plt.imshow(mask)
    plt.show()

    #mask image
    mask2 = np.where((mask==1) + (mask==3), 255, 0).astype('uint8')
    output = cv2.bitwise_and(im, im, mask=mask2)
    plt.figure(figsize=(10,10))
    plt.imshow(output)
    plt.show()

mask_leaf('leaf1.jpg', external_mask=None)
mask_leaf('leaf2.jpg', external_mask=None)

Addressing the external mask. Here's an example of HDBSCAN clustering...I'm not going to go into the details...you can look up the docs and change it or use as-is.

import hdbscan
from collections import Counter


def hdbscan_mask(im_name):

    im = cv2.imread(im_name)
    im = cv2.blur(im, (5,5))

    indices = np.dstack(np.indices(im.shape[:2]))
    data = np.concatenate((indices, im), axis=-1)
    data = data[:,2:]

    data = imb.reshape(im.shape[0]*im.shape[1], 3)
    clusterer = hdbscan.HDBSCAN(min_cluster_size=1000, min_samples=20)
    clusterer.fit(data)

    plt.figure(figsize=(10,10))
    plt.imshow(clusterer.labels_.reshape(im.shape[0:2]))
    plt.show()

    height, width = im.shape[:2]

    mask = np.ones(im.shape[:2], dtype=np.uint8) * 2 #start all possible background
    cv2.circle(mask, (int(width/2.), int(height/2.)), 100, 1, -3) #possible fg

    #grab cluster number for circle
    vals_im = clusterer.labels_.reshape(im.shape[0:2])

    vals = vals_im[mask == 1]
    commonvals = []
    cnts = Counter(vals)
    for v, count in cnts.most_common(20):
    #print '%i: %7d' % (v, count)
    if v == -1:
        continue
    commonvals.append(v)

    tst = np.in1d(vals_im, np.array(commonvals))
    tst = tst.reshape(vals_im.shape)

    hmask = tst.astype(np.uint8)

    plt.figure(figsize=(10,10))
    plt.imshow(hmask)
    plt.show()

    return hmask

hmask = hdbscan_mask('leaf1.jpg')

then to use the initial function with the new mask (output suppressed):

mask_leaf('leaf1.jpg', external_mask=hmask)

This was all made in a notebook from scratch so hopefully there's no errant variables that choke it up when running it somewhere else. (note: I did NOT swap BGR to RGB for plt display, sorry)