Coverage for ckanext/udc/views.py: 13%

1# -*- coding: utf-8 -*-

2from __future__ import annotations

3import logging

5from datetime import datetime

6from collections import OrderedDict

7from functools import partial

8from typing import Any, Iterable, Optional, Union, cast

9from ckanext.udc.graph.logic import get_catalogue_graph

10from werkzeug.datastructures import MultiDict

12from flask import Blueprint

14import ckan.lib.base as base

15from ckan.lib.helpers import helper_functions as h

16from ckan.lib.helpers import Page

17import ckan.lib.navl.dictization_functions as dict_fns

18import ckan.logic as logic

19import ckan.model as model

20import ckan.plugins as plugins

21import ckan.authz as authz

22import ckan.plugins.toolkit as tk

23import ckan.model as model

25from ckan.lib.helpers import Page

26from ckan.common import asbool, current_user, CKANConfig, request, g, config, _

27from ckan.views.dataset import _sort_by, _pager_url, _setup_template_variables, _get_pkg_template

28from ckan.lib.search import SearchQueryError, SearchError

30from ckan.types import Context, Response

32import chalk

33import re

34from urllib.parse import urlencode

35from ckanext.udc.solr.config import get_current_lang

37log = logging.getLogger(__name__)

38bp = Blueprint("catalogue_search", __name__)

42def remove_field(package_type: Optional[str],

43 key: str,

44 value: Optional[str] = None,

45 replace: Optional[str] = None):

46 if not package_type:

47 package_type = u'dataset'

48 url = h.url_for(u'{0}.search'.format(package_type))

50 if key.startswith('extras_'):

51 # Remove the extras_ prefix

52 key = key[7:]

53 elif key.endswith('_ngram'):

54 # Remove the _ngram suffix

55 key = key[:-6]

57 params_items = request.args.items(multi=True)

58 params_nopage = [

59 (k, v) for k, v in params_items

60 if k != 'page'

61 ]

62 params = list(params_nopage)

63 if value:

64 # Assume `fts_` and `exact_` cannot be used together with the same value

65 # print(chalk.red(f"Genererate remove link for Key: {key}, Value: {value}"))

66 if (f"exact_{key}", value) in params:

67 params.remove((f"exact_{key}", value))

68 if (f"fts_{key}", value) in params:

69 params.remove((f"fts_{key}", value))

70 if (key, value) in params:

71 params.remove((key, value))

73 else:

74 for (k, v) in params[:]:

75 if k == key:

76 if (f"exact_{key}", value) in params:

77 params.remove((f"exact_{key}", value))

78 if (f"fts_{key}", value) in params:

79 params.remove((f"fts_{key}", value))

80 if (k, v) in params:

81 params.remove((k, v))

83 if replace is not None:

84 params.append((key, replace))

86 params = [(k, v.encode('utf-8') if isinstance(v, str) else str(v))

87 for k, v in params]

88 return url + u'?' + urlencode(params)

91# Solr type = text_general/text

92# FTS only

93# TODO: If exact match is needed, we need to alter the ckan solr schema

94CKAN_FTS_FIELDS = ["title", "notes", "url", "version",

95 "author", "author_email", "maintainer", "maintainer_email"]

97# Core string facets: exact match; optional _ngram if you enable Option B

98CORE_STRING_FACETS = ["organization", "groups", "license_id", "res_format"]

100def _solr_field_for(param_kind: str, ui_field: str, lang: str,

101 text_fields: set[str]) -> str | None:

102 """

103 Map stable UI field names to concrete Solr fields.

104 param_kind: 'fts' | 'exact' | 'min' | 'max'

105 """

106 # Text maturity fields (multilingual)

107 if ui_field in text_fields:

108 if param_kind == 'fts':

109 return f"{ui_field}_{lang}_txt"

110 if param_kind == 'exact':

111 return f"{ui_field}_{lang}_f"

112 if param_kind in ('min', 'max'):

113 return f"extras_{ui_field}"

114

115 # Tags are multilingual too

116 if ui_field == "tags":

117 if param_kind == 'fts':

118 return f"tags_{lang}_txt"

119 if param_kind == 'exact':

120 return f"tags_{lang}_f"

121

122 # CKAN core text fields: fts only

123 if ui_field in CKAN_FTS_FIELDS:

124 if param_kind == 'fts':

125 return f"{ui_field}_{lang}_txt"

126 # ignore exact_ for these

127 return None

128

129 # Core string facets

130 if ui_field in CORE_STRING_FACETS:

131 if param_kind == 'exact':

132 return ui_field

133 # enable FTS via _ngram (Not supported in current schema)

134 # if param_kind == 'fts':

135 # return f"{ui_field}_ngram"

136

137 if ui_field == 'portal_type':

138 if param_kind == 'exact':

139 return 'extras_portal_type'

140

141 # Everything else (selects / numbers / dates etc) use extras_*

142 if param_kind in ('min', 'max'):

143 return f"extras_{ui_field}"

144 # exact match for non-text -> extras_*

145 if param_kind == 'exact':

146 return f"extras_{ui_field}"

147 # fts on non-text: allow user-entered terms as exact values on extras_*

148 if param_kind == 'fts':

149 return f"extras_{ui_field}"

150

151 return None

152

153

154def _solr_field_for_facet_param(param: str, lang: str,

155 text_fields: set[str]) -> str | None:

156 """Map plain facet query params from the sidebar to concrete Solr fields."""

157 if param.startswith("extras_"):

158 return param

159

160 return _solr_field_for('exact', param, lang, text_fields)

161

162def _get_search_details() -> dict[str, Any]:

163 fq = u''

164

165 # fields_grouped will contain a dict of params containing

166 # a list of values eg {u'tags':[u'tag1', u'tag2']}

167

168 fields = []

169 fields_grouped = {} # key: solr_field -> { ui, values|min|max, fts:bool }

170 filter_logics = {} # key: ui_field -> 'AND' or 'OR'

171 include_undefined = set() # set of solr field names for date/number 'include empty'

172 search_extras: 'MultiDict[str, Any]' = MultiDict()

173

174 udc = plugins.get_plugin('udc')

175 # Get the list of text fields from the udc plugin

176 # Only the text fields support full text search

177 text_fields = set(udc.text_fields)

178

179 # Get the list of date fields from the udc plugin

180 date_fields = udc.date_fields

181

182 lang = request.args.get('lang') or get_current_lang()

183

184

185 # Solr type = string

186 # FTS + exact

187 # For FTS match, we need to add suffix `_ngram` to the field name (ckan solr schema is altered to support this)

188 ckan_fields_exact = ["tags", "organization", "license_id"]

189

190 for (param, value) in request.args.items(multi=True):

191 # Keep the primary query and internal params out of custom facet parsing.

192 if param in [u'q', u'page', u'sort'] or not len(value) or param.startswith(u'_'):

193 continue

194

195 # Ignore internal parameters

196 print(chalk.green(f"Param: {param}, Value: {value}"))

197

198 # Toggle logic

199 if param.startswith('filter-logic-'):

200 ui_name = param[13:]

201 if value.lower() == 'and':

202 filter_logics[ui_name] = 'AND'

203 elif value == 'date':

204 # include undefined date

205 solr_key = _solr_field_for('min', ui_name, lang, text_fields) # same base field

206 if solr_key:

207 include_undefined.add(solr_key)

208 elif value == 'number':

209 solr_key = _solr_field_for('min', ui_name, lang, text_fields)

210 if solr_key:

211 include_undefined.add(solr_key)

212 continue

213

214 # fts_*

215 if param.startswith('fts_'):

216 ui_name = param[4:]

217 solr_key = _solr_field_for('fts', ui_name, lang, text_fields)

218 if not solr_key:

219 continue

220 fields_grouped.setdefault(solr_key, {'ui': ui_name, 'fts': True, 'values': []})

221 fields_grouped[solr_key]['values'].append(value)

222 continue

223

224 # exact_*

225 if param.startswith('exact_'):

226 ui_name = param[6:]

227 solr_key = _solr_field_for('exact', ui_name, lang, text_fields)

228 if not solr_key:

229 continue

230 fields_grouped.setdefault(solr_key, {'ui': ui_name, 'fts': False, 'values': []})

231 fields_grouped[solr_key]['values'].append(value)

232 continue

233

234 # min_/max_ (numbers/dates on extras_*)

235 if param.startswith('min_'):

236 ui_name = param[4:]

237 solr_key = _solr_field_for('min', ui_name, lang, text_fields)

238 if not solr_key:

239 continue

240 fields_grouped.setdefault(solr_key, {'ui': ui_name})

241 fields_grouped[solr_key]['min'] = value

242 continue

243

244 if param.startswith('max_'):

245 ui_name = param[4:]

246 solr_key = _solr_field_for('max', ui_name, lang, text_fields)

247 if not solr_key:

248 continue

249 fields_grouped.setdefault(solr_key, {'ui': ui_name})

250 fields_grouped[solr_key]['max'] = value

251 continue

252

253 # Plain facet params from CKAN sidebar links use stable outward keys.

254 # Convert them into exact-match Solr filters instead of dropping them.

255 solr_key = _solr_field_for_facet_param(param, lang, text_fields)

256 if solr_key:

257 ui_name = param[7:] if param.startswith('extras_') else param

258 fields_grouped.setdefault(solr_key, {'ui': ui_name, 'fts': False, 'values': []})

259 fields_grouped[solr_key]['values'].append(value)

260 continue

261

262 # legacy / unknown -> pass-through as extras

263 if not param.startswith(u'ext_'):

264 fields.append((param, value))

265 else:

266 search_extras.update({param: value})

267

268 # Build fq

269 from datetime import datetime

270 for solr_key, opts in fields_grouped.items():

271 # values group

272 if 'values' in opts:

273 vals = opts['values']

274 ui_name = opts.get('ui', solr_key)

275 logic_op = filter_logics.get(ui_name, 'OR')

276 if len(vals) > 1:

277 joined = f' {logic_op} '.join([f'"{v}"' for v in vals])

278 fq += f' {solr_key}:({joined})'

279 else:

280 fq += f' {solr_key}:"{vals[0]}"'

281 continue

282

283 # range group (dates / numbers on extras_*)

284 _min = opts.get('min')

285 _max = opts.get('max')

286 # Date normalization if the underlying UI field was a date field

287 ui_name = opts.get('ui')

288 if ui_name and solr_key.startswith("extras_") and ui_name in date_fields:

289 try:

290 # Convert date to UTC ISO format

291 if _min:

292 _min = datetime.strptime(_min, '%Y-%m-%d').strftime('%Y-%m-%dT%H:%M:%SZ')

293 if _max:

294 d = datetime.strptime(_max, '%Y-%m-%d')

295 # Add 23:59:59 to the max date to include the whole day

296 _max = d.replace(hour=23, minute=59, second=59).strftime('%Y-%m-%dT%H:%M:%SZ')

297 except ValueError:

298 # If the date is not in the correct format, skip it

299 continue

300

301 # Handle min and max values for number and date ranges

302 if _min and _max:

303 range_query = f' {solr_key}:[{_min} TO {_max}]'

304 elif _min:

305 range_query = f' {solr_key}:[{_min} TO *]'

306 elif _max:

307 range_query = f' {solr_key}:[* TO {_max}]'

308 else:

309 range_query = ""

310

311 # Handle undefined date and number ranges

312 if range_query:

313 if solr_key in include_undefined:

314 range_query = f'({range_query} OR (*:* -{solr_key}:[* TO *]))'

315 fq += range_query

316

317 extras = dict((k, v[0]) if len(v) == 1 else (k, v)

318 for k, v in search_extras.lists())

319

320 return {

321 u'fields': fields,

322 u'fields_grouped': fields_grouped,

323 u'fq': fq,

324 u'search_extras': extras,

325 'filter_logics': filter_logics

326 }

327

328

329def _facet_alias_map(facet_keys: list[str], lang: str) -> tuple[list[str], dict[str, str]]:

330 """

331 Given stable facet keys, return (solr_facet_fields, alias_to_solr_map).

332 - text maturity fields in udc.text_fields -> <name>_<lang>_f

333 - tags -> tags_<lang>_f

334 - extras_* and other core facets -> as-is

335 """

336 udc = plugins.get_plugin('udc')

337 text_fields = set(udc.text_fields or [])

338

339 alias_to_solr = OrderedDict()

340 for key in facet_keys:

341 if key == "tags":

342 alias_to_solr[key] = f"tags_{lang}_f"

343 elif key == "portal_type":

344 alias_to_solr[key] = "extras_portal_type"

345 elif key.startswith("extras_"):

346 alias_to_solr[key] = key

347 elif key in text_fields:

348 alias_to_solr[key] = f"{key}_{lang}_f"

349 else:

350 alias_to_solr[key] = key

351

352 solr_fields = list(dict.fromkeys(alias_to_solr.values()))

353 return solr_fields, alias_to_solr

354

355

356@bp.route(

357 "/catalogue",

358 endpoint="search",

359 strict_slashes=False

360)

361def custom_dataset_search():

362 package_type = 'catalogue'

363 extra_vars: dict[str, Any] = {}

364

365 # unicode format (decoded from utf8)

366 extra_vars[u'q'] = q = request.args.get(u'q', u'')

367

368 extra_vars['query_error'] = False

369 page = h.get_page_number(request.args)

370

371 limit = config.get(u'ckan.datasets_per_page')

372

373 # print(chalk.green(f"Page: {page}, Limit: {limit}, Query: {q}, Package Type: {package_type}, Current User: {current_user.name}"))

374

375 # most search operations should reset the page counter:

376 params_nopage = [(k, v) for k, v in request.args.items(multi=True)

377 if k != u'page']

378

379 # remove_field is a partial function that will remove a field from the search

380 # results. It is used in the search results template to generate links that

381 # remove a field from the search results.

382 extra_vars[u'remove_field'] = partial(remove_field, package_type)

383

384 # print("Remove field: ", extra_vars[u'remove_field'])

385

386 sort_by = request.args.get(u'sort', None)

387 params_nosort = [(k, v) for k, v in params_nopage if k != u'sort']

388

389 extra_vars[u'sort_by'] = partial(_sort_by, params_nosort, package_type)

390 # print("Sort by: ", sort_by)

391

392 if not sort_by:

393 sort_by_fields = []

394 else:

395 sort_by_fields = [field.split()[0] for field in sort_by.split(u',')]

396 extra_vars[u'sort_by_fields'] = sort_by_fields

397

398 pager_url = partial(_pager_url, params_nopage, package_type)

399

400 details = _get_search_details()

401 print(details)

402 extra_vars[u'fields'] = details[u'fields']

403 extra_vars[u'fields_grouped'] = details[u'fields_grouped']

404 extra_vars[u'filter_logics'] = details[u'filter_logics']

405 fq = details[u'fq']

406 search_extras = details[u'search_extras']

407

408 context = cast(Context, {

409 u'model': model,

410 u'session': model.Session,

411 u'user': current_user.name,

412 u'for_view': True,

413 u'auth_user_obj': current_user

414 })

415

416 # Unless changed via config options, don't show other dataset

417 # types any search page. Potential alternatives are do show them

418 # on the default search page (dataset) or on one other search page

419 search_all_type = config.get(u'ckan.search.show_all_types')

420 search_all = False

421

422 try:

423 # If the "type" is set to True or False, convert to bool

424 # and we know that no type was specified, so use traditional

425 # behaviour of applying this only to dataset type

426 search_all = asbool(search_all_type)

427 search_all_type = u'dataset'

428 # Otherwise we treat as a string representing a type

429 except ValueError:

430 search_all = True

431

432 if not search_all or package_type != search_all_type:

433 # Only show datasets of this particular type

434 fq += u' +dataset_type:{type}'.format(type=package_type)

435

436 facets: dict[str, str] = OrderedDict()

437

438 org_label = h.humanize_entity_type(

439 u'organization',

440 h.default_group_type(u'organization'),

441 u'facet label') or _(u'Organizations')

442

443 group_label = h.humanize_entity_type(

444 u'group',

445 h.default_group_type(u'group'),

446 u'facet label') or _(u'Groups')

447

448 default_facet_titles = {

449 u'organization': org_label,

450 u'groups': group_label,

451 u'tags': _(u'Tags'),

452 u'res_format': _(u'Formats'),

453 u'license_id': _(u'Licenses'),

454 }

455

456 for facet in h.facets():

457 if facet in default_facet_titles:

458 facets[facet] = default_facet_titles[facet]

459 else:

460 facets[facet] = facet

461

462 # Facet titles

463 for plugin in plugins.PluginImplementations(plugins.IFacets):

464 facets = plugin.dataset_facets(facets, package_type)

465

466 facet_fields = list(facets.keys())

467 # # Remove date facet as it is not supported by solr

468 # for date_field in h.date_fields:

469 # if date_field in facets:

470 # print("Removing date field: ", date_field)

471 # facet_fields.remove(date_field)

472 # facets.pop(date_field)

473 # print("Facet Fields: ", facet_fields)

474

475 extra_vars[u'facet_titles'] = facets

476 # extra_vars[u'facet_titles'].update(plugins.get_plugin('udc').facet_titles)

477 # print(chalk.yellow(f"Facet Titles: {extra_vars[u'facet_titles']}"))

478

479 lang = request.args.get('lang') or get_current_lang()

480 facet_fields_stable = list(facets.keys())

481 solr_facet_fields, alias_to_solr = _facet_alias_map(facet_fields_stable, lang)

482

483 data_dict: dict[str, Any] = {

484 u'q': q,

485 u'fq': fq.strip(),

486 u'facet.field': solr_facet_fields,

487 # u'facet.limit': -1,

488 u'rows': limit,

489 u'start': (page - 1) * limit,

490 u'sort': sort_by,

491 u'extras': search_extras,

492 u'include_private': config.get(

493 u'ckan.search.default_include_private'),

494 }

495 # print(chalk.green(f"Data Dict: {data_dict}"))

496 try:

497 query = logic.get_action(u'package_search')(context, data_dict)

498

499 extra_vars[u'sort_by_selected'] = query[u'sort']

500

501 extra_vars[u'page'] = Page(

502 collection=query[u'results'],

503 page=page,

504 url=pager_url,

505 item_count=query[u'count'],

506 items_per_page=limit

507 )

508 # print(chalk.red("search_facets"), query[u'search_facets'])

509

510 raw_facets = query.get('search_facets', {})

511 search_facets_stable = {}

512 for alias, solr_name in alias_to_solr.items():

513 if solr_name in raw_facets:

514 search_facets_stable[alias] = raw_facets[solr_name]

515

516 extra_vars[u'search_facets'] = search_facets_stable

517 # extra_vars[u'search_facets'] = query[u'search_facets']

518 extra_vars[u'page'].items = query[u'results']

519 except SearchQueryError as se:

520 # User's search parameters are invalid, in such a way that is not

521 # achievable with the web interface, so return a proper error to

522 # discourage spiders which are the main cause of this.

523 log.info(u'Dataset search query rejected: %r', se.args)

524 base.abort(

525 400,

526 _(u'Invalid search query: {error_message}')

527 .format(error_message=str(se))

528 )

529 except SearchError as se:

530 # May be bad input from the user, but may also be more serious like

531 # bad code causing a SOLR syntax error, or a problem connecting to

532 # SOLR

533 log.error(u'Dataset search error: %r', se.args)

534 extra_vars[u'query_error'] = True

535 extra_vars[u'search_facets'] = {}

536 extra_vars[u'page'] = Page(collection=[])

537

538 # FIXME: try to avoid using global variables

539 g.search_facets_limits = {}

540 default_limit: int = config.get(u'search.facets.default')

541 for facet in cast(Iterable[str], extra_vars[u'search_facets'].keys()):

542 try:

543 limit = int(

544 request.args.get(

545 u'_%s_limit' % facet,

546 default_limit

547 )

548 )

549 except ValueError:

550 base.abort(

551 400,

552 _(u'Parameter u"{parameter_name}" is not '

553 u'an integer').format(parameter_name=u'_%s_limit' % facet)

554 )

555

556 g.search_facets_limits[facet] = limit

557

558 _setup_template_variables(context, {}, package_type=package_type)

559

560 extra_vars[u'dataset_type'] = package_type

561

562 # TODO: remove

563 for key, value in extra_vars.items():

564 setattr(g, key, value)

565

566 # print(chalk.green(f"Extra Vars: {extra_vars}"))

567 return base.render('package/custom_search.html', extra_vars)

568

569

570# Redirect /dataset to /catalogue

571@bp.route(

572 "/dataset",

573 endpoint="redirect-search",

574 strict_slashes=False

575)

576def redirect_to_catalogue_search():

577 # Redirect to the catalogue search page

578 new_path = request.path.replace("/dataset", "/catalogue", 1)

579 query = request.query_string.decode("utf-8") if request.query_string else ""

580 lang = request.environ.get("CKAN_LANG")

581 is_default = request.environ.get("CKAN_LANG_IS_DEFAULT", True)

582 if lang and not is_default and not new_path.startswith(f"/{lang}/"):

583 new_path = f"/{lang}{new_path}"

584 new_url = f"{new_path}?{query}" if query else new_path

585 return tk.redirect_to(new_url)

586

587

588

589# Blueprint for raw graph endpoints

590graph_blueprint = Blueprint('udc_graph', __name__)

591__all__ = ["graph_blueprint"]

592

593

594@graph_blueprint.route('/catalogue/<package_id>/graph')

595@graph_blueprint.route('/catalogue/<package_id>/graph.<format>')

596def package_graph(package_id, format=None):

597 """

598 Return the knowledge graph for a package in raw RDF format.

599

600 URL: /catalogue/<package_id>/graph[.format]

601 """

602 from flask import Response, request, abort

603

604 # Get format from URL extension or query parameter

605 if not format:

606 format = request.args.get('format', 'turtle')

607

608 # Map file extensions to format names

609 format_mapping = {

610 'ttl': 'turtle',

611 'turtle': 'turtle',

612 'jsonld': 'json-ld',

613 'json-ld': 'json-ld',

614 'rdf': 'xml',

615 'xml': 'xml',

616 'n3': 'n3',

617 'nt': 'nt',

618 }

619

620 format = format_mapping.get(format.lower(), format.lower())

621

622 # Validate format

623 valid_formats = {'turtle', 'json-ld', 'xml', 'n3', 'nt', 'pretty-xml'}

624 if format not in valid_formats:

625 abort(400, description=f"Invalid format '{format}'. Must be one of: {', '.join(valid_formats)}")

626

627 # Check if GraphDB is disabled

628 if plugins.get_plugin('udc').disable_graphdb:

629 abort(503, description='Knowledge graph feature is disabled. GraphDB connection is not available.')

630

631 # Check access - use package_show authorization

632 try:

633 tk.check_access('package_show', {'user': current_user.name if current_user else None},

634 {'id': package_id})

635 except tk.NotAuthorized:

636 abort(403, description='Not authorized to access this package')

637 except tk.ObjectNotFound:

638 abort(404, description=f'Package not found: {package_id}')

639

640 # Get the graph

641 try:

642 graph_data = get_catalogue_graph(package_id, format)

643 except ValueError as e:

644 log.error(f"Error retrieving graph for package {package_id}: {str(e)}")

645 abort(404, description=str(e))

646 except Exception as e:

647 log.error(f"Unexpected error retrieving graph for package {package_id}: {str(e)}")

648 abort(500, description='Error occurred while retrieving knowledge graph')

649

650 # Set appropriate content type

651 content_types = {

652 'turtle': 'text/turtle; charset=utf-8',

653 'json-ld': 'application/ld+json; charset=utf-8',

654 'xml': 'application/rdf+xml; charset=utf-8',

655 'pretty-xml': 'application/rdf+xml; charset=utf-8',

656 'n3': 'text/n3; charset=utf-8',

657 'nt': 'application/n-triples; charset=utf-8',

658 }

659

660 content_type = content_types.get(format, 'text/plain; charset=utf-8')

661

662 return Response(graph_data, mimetype=content_type)