validateImageTokens static method

Implementation

static bool validateImageTokens(String prompt, int expectedImageCount) {
  try {
    debugPrint('ImageTokenizer: Validating image tokens - expected: $expectedImageCount');

    // Count image tokens using various patterns
    int imageTokenCount = 0;

    // Check for common image token patterns
    final patterns = [
      RegExp(r'<image>.*?</image>', dotAll: true),
      RegExp(r'<｜image｜>.*?</｜image｜>', dotAll: true),
      RegExp(r'\[IMAGE\].*\[/IMAGE\]', dotAll: true),
      RegExp(r'image_data', caseSensitive: false),
    ];

    for (final pattern in patterns) {
      final matches = pattern.allMatches(prompt);
      imageTokenCount += matches.length;
    }

    // Also check for Base64 image data patterns
    final base64Pattern = RegExp(r'[A-Za-z0-9+/]{100,}={0,2}');
    final base64Matches = base64Pattern.allMatches(prompt);
    if (base64Matches.isNotEmpty) {
      // Estimate that large Base64 strings are likely images
      imageTokenCount += base64Matches.length;
    }

    debugPrint('ImageTokenizer: Found $imageTokenCount image tokens in prompt');

    return imageTokenCount >= expectedImageCount;
  } catch (e) {
    debugPrint('ImageTokenizer: Error validating image tokens - $e');
    return false;
  }
}